2024-12-02 21:15:08,136 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 21:15:08,146 main DEBUG Took 0.008756 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 21:15:08,146 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 21:15:08,146 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 21:15:08,147 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 21:15:08,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,155 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 21:15:08,165 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,167 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,167 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,167 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,168 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,168 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,169 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,169 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,169 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,170 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,170 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,171 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,171 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,171 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,172 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,172 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,173 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,174 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,174 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:15:08,175 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,175 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 21:15:08,176 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:15:08,178 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 21:15:08,180 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 21:15:08,180 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 21:15:08,181 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 21:15:08,182 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 21:15:08,192 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 21:15:08,195 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 21:15:08,197 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 21:15:08,198 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 21:15:08,198 main DEBUG createAppenders(={Console}) 2024-12-02 21:15:08,200 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-02 21:15:08,200 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 21:15:08,200 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-02 21:15:08,201 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 21:15:08,201 main DEBUG OutputStream closed 2024-12-02 21:15:08,202 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 21:15:08,202 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 21:15:08,202 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-02 21:15:08,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 21:15:08,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 21:15:08,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 21:15:08,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 21:15:08,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 21:15:08,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 21:15:08,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 21:15:08,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 21:15:08,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 21:15:08,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 21:15:08,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 21:15:08,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 21:15:08,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 21:15:08,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 21:15:08,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 21:15:08,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 21:15:08,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 21:15:08,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 21:15:08,301 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 21:15:08,301 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-02 21:15:08,302 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 21:15:08,303 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-02T21:15:08,531 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630 2024-12-02 21:15:08,534 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 21:15:08,534 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T21:15:08,541 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-02T21:15:08,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T21:15:08,570 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52, deleteOnExit=true 2024-12-02T21:15:08,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T21:15:08,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/test.cache.data in system properties and HBase conf 2024-12-02T21:15:08,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T21:15:08,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/hadoop.log.dir in system properties and HBase conf 2024-12-02T21:15:08,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T21:15:08,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T21:15:08,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T21:15:08,658 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T21:15:08,741 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T21:15:08,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T21:15:08,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T21:15:08,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T21:15:08,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T21:15:08,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T21:15:08,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T21:15:08,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T21:15:08,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T21:15:08,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T21:15:08,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/nfs.dump.dir in system properties and HBase conf 2024-12-02T21:15:08,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/java.io.tmpdir in system properties and HBase conf 2024-12-02T21:15:08,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T21:15:08,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T21:15:08,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T21:15:09,893 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T21:15:09,975 INFO [Time-limited test {}] log.Log(170): Logging initialized @2459ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T21:15:10,048 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T21:15:10,104 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T21:15:10,125 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T21:15:10,125 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T21:15:10,127 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T21:15:10,140 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T21:15:10,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/hadoop.log.dir/,AVAILABLE} 2024-12-02T21:15:10,144 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T21:15:10,305 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/java.io.tmpdir/jetty-localhost-34803-hadoop-hdfs-3_4_1-tests_jar-_-any-12474978408784030300/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T21:15:10,311 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34803} 2024-12-02T21:15:10,312 INFO [Time-limited test {}] server.Server(415): Started @2796ms 2024-12-02T21:15:10,850 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T21:15:10,859 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T21:15:10,860 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T21:15:10,860 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T21:15:10,861 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T21:15:10,862 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/hadoop.log.dir/,AVAILABLE} 2024-12-02T21:15:10,862 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T21:15:10,961 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/java.io.tmpdir/jetty-localhost-41623-hadoop-hdfs-3_4_1-tests_jar-_-any-8092324079973413499/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T21:15:10,962 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:41623} 2024-12-02T21:15:10,962 INFO [Time-limited test {}] server.Server(415): Started @3446ms 2024-12-02T21:15:11,008 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T21:15:12,012 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52/data/data2/current/BP-855536078-172.17.0.3-1733174109311/current, will proceed with Du for space computation calculation, 2024-12-02T21:15:12,012 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52/data/data1/current/BP-855536078-172.17.0.3-1733174109311/current, will proceed with Du for space computation calculation, 2024-12-02T21:15:12,041 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T21:15:12,088 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x848c24ef7ee7becf with lease ID 0x8f7720dd8960cc29: Processing first storage report for DS-499935e7-24c9-4bf7-abdb-ff5d5d177d84 from datanode DatanodeRegistration(127.0.0.1:39153, datanodeUuid=75b38a8c-0eb4-4a7c-a097-bbbfabaebe5d, infoPort=46247, infoSecurePort=0, ipcPort=42289, storageInfo=lv=-57;cid=testClusterID;nsid=330241541;c=1733174109311) 2024-12-02T21:15:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x848c24ef7ee7becf with lease ID 0x8f7720dd8960cc29: from storage DS-499935e7-24c9-4bf7-abdb-ff5d5d177d84 node DatanodeRegistration(127.0.0.1:39153, datanodeUuid=75b38a8c-0eb4-4a7c-a097-bbbfabaebe5d, infoPort=46247, infoSecurePort=0, ipcPort=42289, storageInfo=lv=-57;cid=testClusterID;nsid=330241541;c=1733174109311), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T21:15:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x848c24ef7ee7becf with lease ID 0x8f7720dd8960cc29: Processing first storage report for DS-b27fc98f-9b49-4a52-a495-3cd64331ac41 from datanode DatanodeRegistration(127.0.0.1:39153, datanodeUuid=75b38a8c-0eb4-4a7c-a097-bbbfabaebe5d, infoPort=46247, infoSecurePort=0, ipcPort=42289, storageInfo=lv=-57;cid=testClusterID;nsid=330241541;c=1733174109311) 2024-12-02T21:15:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x848c24ef7ee7becf with lease ID 0x8f7720dd8960cc29: from storage DS-b27fc98f-9b49-4a52-a495-3cd64331ac41 node DatanodeRegistration(127.0.0.1:39153, datanodeUuid=75b38a8c-0eb4-4a7c-a097-bbbfabaebe5d, infoPort=46247, infoSecurePort=0, ipcPort=42289, storageInfo=lv=-57;cid=testClusterID;nsid=330241541;c=1733174109311), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T21:15:12,180 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630 2024-12-02T21:15:12,245 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52/zookeeper_0, clientPort=53072, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T21:15:12,255 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53072 2024-12-02T21:15:12,263 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:15:12,266 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:15:12,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741825_1001 (size=7) 2024-12-02T21:15:12,873 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0 with version=8 2024-12-02T21:15:12,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/hbase-staging 2024-12-02T21:15:12,952 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T21:15:13,176 INFO [Time-limited test {}] client.ConnectionUtils(128): master/87c3fdb6c570:0 server-side Connection retries=45 2024-12-02T21:15:13,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:15:13,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T21:15:13,190 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T21:15:13,190 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:15:13,190 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T21:15:13,310 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T21:15:13,359 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T21:15:13,367 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T21:15:13,371 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T21:15:13,393 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4406 (auto-detected) 2024-12-02T21:15:13,394 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-02T21:15:13,411 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39577 2024-12-02T21:15:13,430 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39577 connecting to ZooKeeper ensemble=127.0.0.1:53072 2024-12-02T21:15:13,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:395770x0, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T21:15:13,565 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39577-0x10197e62b900000 connected 2024-12-02T21:15:13,752 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:15:13,756 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:15:13,767 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:15:13,771 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0, hbase.cluster.distributed=false 2024-12-02T21:15:13,792 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T21:15:13,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39577 2024-12-02T21:15:13,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39577 2024-12-02T21:15:13,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39577 2024-12-02T21:15:13,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39577 2024-12-02T21:15:13,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39577 2024-12-02T21:15:13,891 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/87c3fdb6c570:0 server-side Connection retries=45 2024-12-02T21:15:13,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:15:13,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T21:15:13,893 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T21:15:13,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:15:13,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T21:15:13,895 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T21:15:13,898 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T21:15:13,899 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35563 2024-12-02T21:15:13,900 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35563 connecting to ZooKeeper ensemble=127.0.0.1:53072 2024-12-02T21:15:13,902 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:15:13,904 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:15:13,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355630x0, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T21:15:13,917 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:15:13,917 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35563-0x10197e62b900001 connected 2024-12-02T21:15:13,921 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T21:15:13,927 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T21:15:13,930 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T21:15:13,934 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T21:15:13,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35563 2024-12-02T21:15:13,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35563 2024-12-02T21:15:13,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35563 2024-12-02T21:15:13,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35563 2024-12-02T21:15:13,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35563 2024-12-02T21:15:13,952 DEBUG [M:0;87c3fdb6c570:39577 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;87c3fdb6c570:39577 2024-12-02T21:15:13,953 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:13,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:15:13,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:15:13,972 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:14,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T21:15:14,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:14,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:14,002 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T21:15:14,003 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/87c3fdb6c570,39577,1733174113031 from backup master directory 2024-12-02T21:15:14,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:14,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:15:14,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:15:14,012 WARN [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T21:15:14,012 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:14,014 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T21:15:14,015 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T21:15:14,064 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/hbase.id] with ID: 1e0afb41-015e-46ad-ac26-3e94247c4bda 2024-12-02T21:15:14,065 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/.tmp/hbase.id 2024-12-02T21:15:14,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741826_1002 (size=42) 2024-12-02T21:15:14,075 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/.tmp/hbase.id]:[hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/hbase.id] 2024-12-02T21:15:14,120 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:15:14,125 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T21:15:14,142 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-02T21:15:14,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:14,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741827_1003 (size=196) 2024-12-02T21:15:14,188 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:15:14,190 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T21:15:14,205 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:14,209 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T21:15:14,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741828_1004 (size=1189) 2024-12-02T21:15:14,652 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store 2024-12-02T21:15:14,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741829_1005 (size=34) 2024-12-02T21:15:15,078 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T21:15:15,085 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:15:15,086 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T21:15:15,086 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:15:15,086 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:15:15,088 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T21:15:15,088 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:15:15,088 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:15:15,090 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733174115086Disabling compacts and flushes for region at 1733174115086Disabling writes for close at 1733174115088 (+2 ms)Writing region close event to WAL at 1733174115088Closed at 1733174115088 2024-12-02T21:15:15,092 WARN [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/.initializing 2024-12-02T21:15:15,092 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/WALs/87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:15,100 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T21:15:15,114 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87c3fdb6c570%2C39577%2C1733174113031, suffix=, logDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/WALs/87c3fdb6c570,39577,1733174113031, archiveDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/oldWALs, maxLogs=10 2024-12-02T21:15:15,141 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/WALs/87c3fdb6c570,39577,1733174113031/87c3fdb6c570%2C39577%2C1733174113031.1733174115119, exclude list is [], retry=0 2024-12-02T21:15:15,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39153,DS-499935e7-24c9-4bf7-abdb-ff5d5d177d84,DISK] 2024-12-02T21:15:15,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T21:15:15,188 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/WALs/87c3fdb6c570,39577,1733174113031/87c3fdb6c570%2C39577%2C1733174113031.1733174115119 2024-12-02T21:15:15,189 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46247:46247)] 2024-12-02T21:15:15,190 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:15:15,190 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:15:15,193 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,194 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,228 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T21:15:15,253 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:15,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:15:15,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T21:15:15,258 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:15,259 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:15,259 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T21:15:15,262 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:15,263 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:15,263 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,266 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T21:15:15,266 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:15,267 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:15,267 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,270 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,272 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,276 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,277 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,280 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T21:15:15,284 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:15:15,288 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:15:15,289 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64131061, jitterRate=-0.04437272250652313}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T21:15:15,293 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733174115205Initializing all the Stores at 1733174115207 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174115207Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174115208 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174115208Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174115208Cleaning up temporary data from old regions at 1733174115277 (+69 ms)Region opened successfully at 1733174115293 (+16 ms) 2024-12-02T21:15:15,295 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T21:15:15,325 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ec1fb41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87c3fdb6c570/172.17.0.3:0 2024-12-02T21:15:15,350 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T21:15:15,359 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T21:15:15,359 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T21:15:15,361 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:15:15,362 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T21:15:15,366 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-02T21:15:15,367 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T21:15:15,387 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T21:15:15,394 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T21:15:15,474 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T21:15:15,477 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T21:15:15,480 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T21:15:15,491 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T21:15:15,494 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T21:15:15,499 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T21:15:15,505 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T21:15:15,507 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T21:15:15,516 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T21:15:15,531 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T21:15:15,544 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T21:15:15,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T21:15:15,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T21:15:15,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:15,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:15,562 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=87c3fdb6c570,39577,1733174113031, sessionid=0x10197e62b900000, setting cluster-up flag (Was=false) 2024-12-02T21:15:15,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:15,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:15,621 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T21:15:15,624 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:15,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:15,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:15,685 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T21:15:15,690 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:15,698 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T21:15:15,745 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(746): ClusterId : 1e0afb41-015e-46ad-ac26-3e94247c4bda 2024-12-02T21:15:15,747 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T21:15:15,760 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T21:15:15,760 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T21:15:15,763 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:15:15,770 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T21:15:15,770 DEBUG [RS:0;87c3fdb6c570:35563 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335ab8f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87c3fdb6c570/172.17.0.3:0 2024-12-02T21:15:15,772 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T21:15:15,778 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T21:15:15,783 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;87c3fdb6c570:35563 2024-12-02T21:15:15,783 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 87c3fdb6c570,39577,1733174113031 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T21:15:15,785 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T21:15:15,786 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T21:15:15,786 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T21:15:15,788 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(2659): reportForDuty to master=87c3fdb6c570,39577,1733174113031 with port=35563, startcode=1733174113859 2024-12-02T21:15:15,790 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/87c3fdb6c570:0, corePoolSize=5, maxPoolSize=5 2024-12-02T21:15:15,790 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/87c3fdb6c570:0, corePoolSize=5, maxPoolSize=5 2024-12-02T21:15:15,790 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=5, maxPoolSize=5 2024-12-02T21:15:15,790 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=5, maxPoolSize=5 2024-12-02T21:15:15,790 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/87c3fdb6c570:0, corePoolSize=10, maxPoolSize=10 2024-12-02T21:15:15,790 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,791 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=2, maxPoolSize=2 2024-12-02T21:15:15,791 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,792 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174145792 2024-12-02T21:15:15,793 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T21:15:15,794 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T21:15:15,796 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T21:15:15,797 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T21:15:15,797 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T21:15:15,797 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T21:15:15,798 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T21:15:15,798 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T21:15:15,799 DEBUG [RS:0;87c3fdb6c570:35563 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T21:15:15,798 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,801 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T21:15:15,802 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T21:15:15,803 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T21:15:15,804 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:15,804 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T21:15:15,805 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T21:15:15,805 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T21:15:15,807 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.large.0-1733174115806,5,FailOnTimeoutGroup] 2024-12-02T21:15:15,808 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.small.0-1733174115807,5,FailOnTimeoutGroup] 2024-12-02T21:15:15,808 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,808 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T21:15:15,810 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,810 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741831_1007 (size=1321) 2024-12-02T21:15:15,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48619, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T21:15:15,840 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:15,841 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39577 {}] master.ServerManager(517): Registering regionserver=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:15,853 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0 2024-12-02T21:15:15,854 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46549 2024-12-02T21:15:15,854 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T21:15:15,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T21:15:15,864 DEBUG [RS:0;87c3fdb6c570:35563 {}] zookeeper.ZKUtil(111): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:15,864 WARN [RS:0;87c3fdb6c570:35563 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T21:15:15,865 INFO [RS:0;87c3fdb6c570:35563 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T21:15:15,865 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/WALs/87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:15,867 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [87c3fdb6c570,35563,1733174113859] 2024-12-02T21:15:15,887 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T21:15:15,899 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T21:15:15,904 INFO [RS:0;87c3fdb6c570:35563 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T21:15:15,904 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,905 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T21:15:15,910 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T21:15:15,911 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,911 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,911 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,911 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,911 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,912 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,912 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0, corePoolSize=2, maxPoolSize=2 2024-12-02T21:15:15,912 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,912 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,912 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,913 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,913 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,913 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:15:15,913 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/87c3fdb6c570:0, corePoolSize=3, maxPoolSize=3 2024-12-02T21:15:15,914 DEBUG [RS:0;87c3fdb6c570:35563 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0, corePoolSize=3, maxPoolSize=3 2024-12-02T21:15:15,915 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,915 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,915 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,915 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,915 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,916 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,35563,1733174113859-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T21:15:15,932 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T21:15:15,934 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,35563,1733174113859-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,934 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,935 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.Replication(171): 87c3fdb6c570,35563,1733174113859 started 2024-12-02T21:15:15,950 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:15,950 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1482): Serving as 87c3fdb6c570,35563,1733174113859, RpcServer on 87c3fdb6c570/172.17.0.3:35563, sessionid=0x10197e62b900001 2024-12-02T21:15:15,951 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T21:15:15,951 DEBUG [RS:0;87c3fdb6c570:35563 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:15,951 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87c3fdb6c570,35563,1733174113859' 2024-12-02T21:15:15,952 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T21:15:15,953 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T21:15:15,953 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T21:15:15,953 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T21:15:15,953 DEBUG [RS:0;87c3fdb6c570:35563 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:15,954 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87c3fdb6c570,35563,1733174113859' 2024-12-02T21:15:15,954 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T21:15:15,954 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T21:15:15,955 DEBUG [RS:0;87c3fdb6c570:35563 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T21:15:15,955 INFO [RS:0;87c3fdb6c570:35563 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T21:15:15,955 INFO [RS:0;87c3fdb6c570:35563 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T21:15:16,065 INFO [RS:0;87c3fdb6c570:35563 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T21:15:16,068 INFO [RS:0;87c3fdb6c570:35563 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87c3fdb6c570%2C35563%2C1733174113859, suffix=, logDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/WALs/87c3fdb6c570,35563,1733174113859, archiveDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/oldWALs, maxLogs=32 2024-12-02T21:15:16,081 DEBUG [RS:0;87c3fdb6c570:35563 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/WALs/87c3fdb6c570,35563,1733174113859/87c3fdb6c570%2C35563%2C1733174113859.1733174116070, exclude list is [], retry=0 2024-12-02T21:15:16,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39153,DS-499935e7-24c9-4bf7-abdb-ff5d5d177d84,DISK] 2024-12-02T21:15:16,088 INFO [RS:0;87c3fdb6c570:35563 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/WALs/87c3fdb6c570,35563,1733174113859/87c3fdb6c570%2C35563%2C1733174113859.1733174116070 2024-12-02T21:15:16,089 DEBUG [RS:0;87c3fdb6c570:35563 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46247:46247)] 2024-12-02T21:15:16,219 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T21:15:16,220 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0 2024-12-02T21:15:16,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741833_1009 (size=32) 2024-12-02T21:15:16,635 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:15:16,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T21:15:16,641 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T21:15:16,641 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:16,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:15:16,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T21:15:16,646 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T21:15:16,646 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:16,647 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:15:16,647 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T21:15:16,649 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T21:15:16,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:16,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:15:16,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T21:15:16,653 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T21:15:16,653 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:16,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:15:16,654 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T21:15:16,655 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740 2024-12-02T21:15:16,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740 2024-12-02T21:15:16,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T21:15:16,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T21:15:16,660 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:15:16,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T21:15:16,665 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:15:16,666 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63383361, jitterRate=-0.055514320731163025}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:15:16,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733174116636Initializing all the Stores at 1733174116638 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174116638Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174116638Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174116638Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174116638Cleaning up temporary data from old regions at 1733174116659 (+21 ms)Region opened successfully at 1733174116667 (+8 ms) 2024-12-02T21:15:16,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T21:15:16,667 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T21:15:16,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T21:15:16,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T21:15:16,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T21:15:16,669 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T21:15:16,669 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733174116667Disabling compacts and flushes for region at 1733174116667Disabling writes for close at 1733174116668 (+1 ms)Writing region close event to WAL at 1733174116668Closed at 1733174116669 (+1 ms) 2024-12-02T21:15:16,672 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T21:15:16,672 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T21:15:16,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T21:15:16,683 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:15:16,686 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T21:15:16,839 DEBUG [87c3fdb6c570:39577 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T21:15:16,852 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:16,859 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,35563,1733174113859, state=OPENING 2024-12-02T21:15:16,916 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T21:15:16,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:16,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:15:16,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:15:16,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:15:16,929 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:15:16,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:15:17,113 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T21:15:17,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38765, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T21:15:17,129 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T21:15:17,130 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T21:15:17,131 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T21:15:17,134 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87c3fdb6c570%2C35563%2C1733174113859.meta, suffix=.meta, logDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/WALs/87c3fdb6c570,35563,1733174113859, archiveDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/oldWALs, maxLogs=32 2024-12-02T21:15:17,153 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/WALs/87c3fdb6c570,35563,1733174113859/87c3fdb6c570%2C35563%2C1733174113859.meta.1733174117136.meta, exclude list is [], retry=0 2024-12-02T21:15:17,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39153,DS-499935e7-24c9-4bf7-abdb-ff5d5d177d84,DISK] 2024-12-02T21:15:17,159 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/WALs/87c3fdb6c570,35563,1733174113859/87c3fdb6c570%2C35563%2C1733174113859.meta.1733174117136.meta 2024-12-02T21:15:17,160 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46247:46247)] 2024-12-02T21:15:17,160 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:15:17,162 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T21:15:17,164 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T21:15:17,168 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T21:15:17,171 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T21:15:17,172 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:15:17,172 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T21:15:17,172 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T21:15:17,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T21:15:17,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T21:15:17,177 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:17,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:15:17,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T21:15:17,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T21:15:17,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:17,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:15:17,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T21:15:17,182 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T21:15:17,182 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:17,183 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:15:17,183 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T21:15:17,185 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T21:15:17,185 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:17,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:15:17,186 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T21:15:17,187 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740 2024-12-02T21:15:17,190 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740 2024-12-02T21:15:17,192 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T21:15:17,192 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T21:15:17,193 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:15:17,196 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T21:15:17,198 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61127561, jitterRate=-0.08912836015224457}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:15:17,198 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T21:15:17,199 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733174117173Writing region info on filesystem at 1733174117173Initializing all the Stores at 1733174117175 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174117175Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174117175Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174117175Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174117175Cleaning up temporary data from old regions at 1733174117193 (+18 ms)Running coprocessor post-open hooks at 1733174117198 (+5 ms)Region opened successfully at 1733174117199 (+1 ms) 2024-12-02T21:15:17,206 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733174117103 2024-12-02T21:15:17,216 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T21:15:17,217 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T21:15:17,218 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:17,220 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,35563,1733174113859, state=OPEN 2024-12-02T21:15:17,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:15:17,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:15:17,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:15:17,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:15:17,256 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:17,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T21:15:17,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,35563,1733174113859 in 325 msec 2024-12-02T21:15:17,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T21:15:17,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 588 msec 2024-12-02T21:15:17,271 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T21:15:17,271 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T21:15:17,291 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:17,292 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:17,311 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:17,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44761, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:17,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6060 sec 2024-12-02T21:15:17,334 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733174117334, completionTime=-1 2024-12-02T21:15:17,338 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T21:15:17,338 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:15:17,362 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T21:15:17,362 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174177362 2024-12-02T21:15:17,362 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174237362 2024-12-02T21:15:17,362 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-02T21:15:17,364 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,39577,1733174113031-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:17,364 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,39577,1733174113031-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:17,364 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,39577,1733174113031-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:17,365 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-87c3fdb6c570:39577, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:17,366 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:17,366 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:17,371 DEBUG [master/87c3fdb6c570:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T21:15:17,389 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.377sec 2024-12-02T21:15:17,390 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T21:15:17,392 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T21:15:17,393 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T21:15:17,393 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T21:15:17,393 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T21:15:17,394 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,39577,1733174113031-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T21:15:17,394 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,39577,1733174113031-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T21:15:17,402 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T21:15:17,403 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T21:15:17,403 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,39577,1733174113031-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:15:17,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:17,461 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T21:15:17,461 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T21:15:17,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:17,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:17,473 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db5a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:17,483 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:17,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:17,486 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:17,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:17,513 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:17,523 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=545, ProcessCount=11, AvailableMemoryMB=2745 2024-12-02T21:15:17,530 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T21:15:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27423f7d 2024-12-02T21:15:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T21:15:17,557 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T21:15:17,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T21:15:17,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:15:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:17,582 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T21:15:17,585 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:17,588 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T21:15:17,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-02T21:15:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:15:17,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741835_1011 (size=1061) 2024-12-02T21:15:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:15:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:15:18,011 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0 2024-12-02T21:15:18,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741836_1012 (size=53) 2024-12-02T21:15:18,023 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:15:18,023 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8a7bf3289fb638014baa7fb12fa0d82c, disabling compactions & flushes 2024-12-02T21:15:18,024 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:18,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:18,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. after waiting 0 ms 2024-12-02T21:15:18,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:18,024 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:18,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8a7bf3289fb638014baa7fb12fa0d82c: Waiting for close lock at 1733174118023Disabling compacts and flushes for region at 1733174118023Disabling writes for close at 1733174118024 (+1 ms)Writing region close event to WAL at 1733174118024Closed at 1733174118024 2024-12-02T21:15:18,027 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T21:15:18,033 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733174118027"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174118027"}]},"ts":"1733174118027"} 2024-12-02T21:15:18,038 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T21:15:18,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T21:15:18,042 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174118040"}]},"ts":"1733174118040"} 2024-12-02T21:15:18,047 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T21:15:18,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c, ASSIGN}] 2024-12-02T21:15:18,051 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c, ASSIGN 2024-12-02T21:15:18,053 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c, ASSIGN; state=OFFLINE, location=87c3fdb6c570,35563,1733174113859; forceNewPlan=false, retain=false 2024-12-02T21:15:18,207 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8a7bf3289fb638014baa7fb12fa0d82c, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:18,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c, ASSIGN because future has completed 2024-12-02T21:15:18,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:15:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:15:18,380 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:18,381 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:15:18,381 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,381 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:15:18,381 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,381 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,384 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,387 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:15:18,387 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a7bf3289fb638014baa7fb12fa0d82c columnFamilyName A 2024-12-02T21:15:18,388 DEBUG [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:18,389 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] regionserver.HStore(327): Store=8a7bf3289fb638014baa7fb12fa0d82c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:18,389 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,391 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:15:18,392 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a7bf3289fb638014baa7fb12fa0d82c columnFamilyName B 2024-12-02T21:15:18,392 DEBUG [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:18,393 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] regionserver.HStore(327): Store=8a7bf3289fb638014baa7fb12fa0d82c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:18,393 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,396 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:15:18,396 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a7bf3289fb638014baa7fb12fa0d82c columnFamilyName C 2024-12-02T21:15:18,397 DEBUG [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:18,398 INFO [StoreOpener-8a7bf3289fb638014baa7fb12fa0d82c-1 {}] regionserver.HStore(327): Store=8a7bf3289fb638014baa7fb12fa0d82c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:18,398 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:18,398 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,400 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,400 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,402 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,402 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,403 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:15:18,406 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,409 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:15:18,410 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8a7bf3289fb638014baa7fb12fa0d82c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71629029, jitterRate=0.06735570728778839}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:15:18,411 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,411 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8a7bf3289fb638014baa7fb12fa0d82c: Running coprocessor pre-open hook at 1733174118381Writing region info on filesystem at 1733174118381Initializing all the Stores at 1733174118383 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174118383Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174118383Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174118383Cleaning up temporary data from old regions at 1733174118402 (+19 ms)Running coprocessor post-open hooks at 1733174118411 (+9 ms)Region opened successfully at 1733174118411 2024-12-02T21:15:18,413 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., pid=6, masterSystemTime=1733174118370 2024-12-02T21:15:18,417 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:18,417 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:18,419 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8a7bf3289fb638014baa7fb12fa0d82c, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:18,421 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39577 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,35563,1733174113859, table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:15:18,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:15:18,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T21:15:18,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 in 209 msec 2024-12-02T21:15:18,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T21:15:18,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c, ASSIGN in 381 msec 2024-12-02T21:15:18,436 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T21:15:18,437 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174118436"}]},"ts":"1733174118436"} 2024-12-02T21:15:18,440 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T21:15:18,442 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T21:15:18,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 873 msec 2024-12-02T21:15:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:15:18,742 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:18,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,750 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c34a958, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,753 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127234f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,755 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,762 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T21:15:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42827a35 2024-12-02T21:15:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T21:15:18,768 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T21:15:18,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,774 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,777 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,781 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395f1c25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,785 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,789 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1aa4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,793 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c5f854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,796 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51268a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,800 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1768b9d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,802 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,803 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61fba9b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8d6bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,810 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d944a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,813 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ae337, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,816 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796fbe9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d388d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,822 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c24d9c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,825 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:18,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:18,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:18,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b2908d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:18,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:18,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:18,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc72606, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:18,837 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,838 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,839 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,842 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,844 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,847 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:15:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:15:18,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:15:18,855 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,856 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:15:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:15:18,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:15:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:15:18,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,865 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:18,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:15:18,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:18,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:15:18,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-02T21:15:18,891 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:18,894 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:18,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T21:15:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:18,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:15:18,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:18,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:18,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:18,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:18,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:18,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T21:15:19,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/94bf6af8867d4d6ca83d3012c8dbb31f is 50, key is test_row_0/A:col10/1733174118910/Put/seqid=0 2024-12-02T21:15:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174179020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174179020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741837_1013 (size=12001) 2024-12-02T21:15:19,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/94bf6af8867d4d6ca83d3012c8dbb31f 2024-12-02T21:15:19,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T21:15:19,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174179031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:19,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:19,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:19,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:19,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174179057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:19,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174179057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:19,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/f5a9ccbc0bc043848a4b197a6acee5d8 is 50, key is test_row_0/B:col10/1733174118910/Put/seqid=0 2024-12-02T21:15:19,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741838_1014 (size=12001) 2024-12-02T21:15:19,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/f5a9ccbc0bc043848a4b197a6acee5d8 2024-12-02T21:15:19,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174179210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T21:15:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174179212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174179212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174179212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174179213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/d6bf36addb8846039e05bb47c8c30aa4 is 50, key is test_row_0/C:col10/1733174118910/Put/seqid=0 2024-12-02T21:15:19,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T21:15:19,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:19,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:19,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:19,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:19,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:19,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741839_1015 (size=12001) 2024-12-02T21:15:19,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/d6bf36addb8846039e05bb47c8c30aa4 2024-12-02T21:15:19,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/94bf6af8867d4d6ca83d3012c8dbb31f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/94bf6af8867d4d6ca83d3012c8dbb31f 2024-12-02T21:15:19,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/94bf6af8867d4d6ca83d3012c8dbb31f, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T21:15:19,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/f5a9ccbc0bc043848a4b197a6acee5d8 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f5a9ccbc0bc043848a4b197a6acee5d8 2024-12-02T21:15:19,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f5a9ccbc0bc043848a4b197a6acee5d8, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T21:15:19,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/d6bf36addb8846039e05bb47c8c30aa4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d6bf36addb8846039e05bb47c8c30aa4 2024-12-02T21:15:19,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d6bf36addb8846039e05bb47c8c30aa4, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T21:15:19,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8a7bf3289fb638014baa7fb12fa0d82c in 450ms, sequenceid=13, compaction requested=false 2024-12-02T21:15:19,371 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T21:15:19,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:19,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T21:15:19,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:19,427 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T21:15:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:19,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:19,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/7f8409385b0545b2858e3b2c0758c648 is 50, key is test_row_0/A:col10/1733174119045/Put/seqid=0 2024-12-02T21:15:19,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174179459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174179465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174179465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174179469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174179469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741840_1016 (size=12001) 2024-12-02T21:15:19,486 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/7f8409385b0545b2858e3b2c0758c648 2024-12-02T21:15:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/cbd7c09e6af84cf1b5dc43fb77f77ffe is 50, key is test_row_0/B:col10/1733174119045/Put/seqid=0 2024-12-02T21:15:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T21:15:19,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741841_1017 (size=12001) 2024-12-02T21:15:19,547 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/cbd7c09e6af84cf1b5dc43fb77f77ffe 2024-12-02T21:15:19,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/209aa294117e4e87a756dd18d2a5470b is 50, key is test_row_0/C:col10/1733174119045/Put/seqid=0 2024-12-02T21:15:19,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174179580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174179590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741842_1018 (size=12001) 2024-12-02T21:15:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174179591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174179591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174179591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,604 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/209aa294117e4e87a756dd18d2a5470b 2024-12-02T21:15:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/7f8409385b0545b2858e3b2c0758c648 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/7f8409385b0545b2858e3b2c0758c648 2024-12-02T21:15:19,643 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/7f8409385b0545b2858e3b2c0758c648, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T21:15:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/cbd7c09e6af84cf1b5dc43fb77f77ffe as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/cbd7c09e6af84cf1b5dc43fb77f77ffe 2024-12-02T21:15:19,666 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/cbd7c09e6af84cf1b5dc43fb77f77ffe, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T21:15:19,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/209aa294117e4e87a756dd18d2a5470b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/209aa294117e4e87a756dd18d2a5470b 2024-12-02T21:15:19,687 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/209aa294117e4e87a756dd18d2a5470b, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T21:15:19,690 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8a7bf3289fb638014baa7fb12fa0d82c in 263ms, sequenceid=33, compaction requested=false 2024-12-02T21:15:19,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:19,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:19,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-02T21:15:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-02T21:15:19,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T21:15:19,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 799 msec 2024-12-02T21:15:19,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 822 msec 2024-12-02T21:15:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:19,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T21:15:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:19,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/fcbcbc76e90d4a5fb1a068fd7909d64e is 50, key is test_row_0/A:col10/1733174119452/Put/seqid=0 2024-12-02T21:15:19,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741843_1019 (size=12001) 2024-12-02T21:15:19,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174179871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174179872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174179876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174179880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174179884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174179990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:19,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174179990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174180000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174180000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174180004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T21:15:20,040 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:20,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-02T21:15:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T21:15:20,045 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:20,047 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:20,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T21:15:20,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T21:15:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:20,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:20,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:20,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174180212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174180213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174180213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174180213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174180224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/fcbcbc76e90d4a5fb1a068fd7909d64e 2024-12-02T21:15:20,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1916bc7cbbcc4e498046b49e325824aa is 50, key is test_row_0/B:col10/1733174119452/Put/seqid=0 2024-12-02T21:15:20,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741844_1020 (size=12001) 2024-12-02T21:15:20,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1916bc7cbbcc4e498046b49e325824aa 2024-12-02T21:15:20,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/943e562c6d204f608fe53a603631f495 is 50, key is test_row_0/C:col10/1733174119452/Put/seqid=0 2024-12-02T21:15:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T21:15:20,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T21:15:20,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:20,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:20,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:20,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:20,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:20,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741845_1021 (size=12001) 2024-12-02T21:15:20,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/943e562c6d204f608fe53a603631f495 2024-12-02T21:15:20,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/fcbcbc76e90d4a5fb1a068fd7909d64e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fcbcbc76e90d4a5fb1a068fd7909d64e 2024-12-02T21:15:20,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fcbcbc76e90d4a5fb1a068fd7909d64e, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T21:15:20,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1916bc7cbbcc4e498046b49e325824aa as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1916bc7cbbcc4e498046b49e325824aa 2024-12-02T21:15:20,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1916bc7cbbcc4e498046b49e325824aa, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T21:15:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/943e562c6d204f608fe53a603631f495 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/943e562c6d204f608fe53a603631f495 2024-12-02T21:15:20,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/943e562c6d204f608fe53a603631f495, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T21:15:20,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8a7bf3289fb638014baa7fb12fa0d82c in 644ms, sequenceid=46, compaction requested=true 2024-12-02T21:15:20,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:20,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:20,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:20,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:20,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:20,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:20,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:20,453 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:20,455 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:20,458 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:20,460 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:20,460 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:20,460 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:20,460 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:20,461 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f5a9ccbc0bc043848a4b197a6acee5d8, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/cbd7c09e6af84cf1b5dc43fb77f77ffe, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1916bc7cbbcc4e498046b49e325824aa] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=35.2 K 2024-12-02T21:15:20,461 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:20,462 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/94bf6af8867d4d6ca83d3012c8dbb31f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/7f8409385b0545b2858e3b2c0758c648, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fcbcbc76e90d4a5fb1a068fd7909d64e] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=35.2 K 2024-12-02T21:15:20,463 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a9ccbc0bc043848a4b197a6acee5d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733174118886 2024-12-02T21:15:20,464 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94bf6af8867d4d6ca83d3012c8dbb31f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733174118886 2024-12-02T21:15:20,464 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd7c09e6af84cf1b5dc43fb77f77ffe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733174119013 2024-12-02T21:15:20,465 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f8409385b0545b2858e3b2c0758c648, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733174119013 2024-12-02T21:15:20,465 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 1916bc7cbbcc4e498046b49e325824aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174119452 2024-12-02T21:15:20,466 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcbcbc76e90d4a5fb1a068fd7909d64e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174119452 2024-12-02T21:15:20,499 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#9 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:20,502 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/e7220d0952e7483685da62dd17d206e1 is 50, key is test_row_0/B:col10/1733174119452/Put/seqid=0 2024-12-02T21:15:20,510 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#10 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:20,511 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/275b560ad33542f184c9fbc850676f5c is 50, key is test_row_0/A:col10/1733174119452/Put/seqid=0 2024-12-02T21:15:20,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T21:15:20,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:20,524 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T21:15:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:20,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:20,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741846_1022 (size=12104) 2024-12-02T21:15:20,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741847_1023 (size=12104) 2024-12-02T21:15:20,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/3b53270d8eaa40538ae841818e62f871 is 50, key is test_row_0/A:col10/1733174119871/Put/seqid=0 2024-12-02T21:15:20,567 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/275b560ad33542f184c9fbc850676f5c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/275b560ad33542f184c9fbc850676f5c 2024-12-02T21:15:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174180564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174180567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174180570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174180571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174180577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741848_1024 (size=12001) 2024-12-02T21:15:20,608 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/3b53270d8eaa40538ae841818e62f871 2024-12-02T21:15:20,617 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into 275b560ad33542f184c9fbc850676f5c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:20,618 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:20,620 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=13, startTime=1733174120445; duration=0sec 2024-12-02T21:15:20,621 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:20,621 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:20,621 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:20,626 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:20,627 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:20,627 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:20,627 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d6bf36addb8846039e05bb47c8c30aa4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/209aa294117e4e87a756dd18d2a5470b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/943e562c6d204f608fe53a603631f495] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=35.2 K 2024-12-02T21:15:20,631 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6bf36addb8846039e05bb47c8c30aa4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733174118886 2024-12-02T21:15:20,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/d578fd2204d04a16a997ce5ff44f8780 is 50, key is test_row_0/B:col10/1733174119871/Put/seqid=0 2024-12-02T21:15:20,635 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 209aa294117e4e87a756dd18d2a5470b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733174119013 2024-12-02T21:15:20,637 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 943e562c6d204f608fe53a603631f495, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174119452 2024-12-02T21:15:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T21:15:20,673 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:20,673 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/d84ab05e9b1149639de6f250d134ffee is 50, key is test_row_0/C:col10/1733174119452/Put/seqid=0 2024-12-02T21:15:20,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741849_1025 (size=12001) 2024-12-02T21:15:20,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174180689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174180690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174180690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174180701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,702 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/d578fd2204d04a16a997ce5ff44f8780 2024-12-02T21:15:20,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174180701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741850_1026 (size=12104) 2024-12-02T21:15:20,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/3475bfa9db064ec48375e48f61496bfb is 50, key is test_row_0/C:col10/1733174119871/Put/seqid=0 2024-12-02T21:15:20,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741851_1027 (size=12001) 2024-12-02T21:15:20,753 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/3475bfa9db064ec48375e48f61496bfb 2024-12-02T21:15:20,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/3b53270d8eaa40538ae841818e62f871 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3b53270d8eaa40538ae841818e62f871 2024-12-02T21:15:20,781 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3b53270d8eaa40538ae841818e62f871, entries=150, sequenceid=65, filesize=11.7 K 2024-12-02T21:15:20,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/d578fd2204d04a16a997ce5ff44f8780 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/d578fd2204d04a16a997ce5ff44f8780 2024-12-02T21:15:20,801 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/d578fd2204d04a16a997ce5ff44f8780, entries=150, sequenceid=65, filesize=11.7 K 2024-12-02T21:15:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/3475bfa9db064ec48375e48f61496bfb as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3475bfa9db064ec48375e48f61496bfb 2024-12-02T21:15:20,826 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3475bfa9db064ec48375e48f61496bfb, entries=150, sequenceid=65, filesize=11.7 K 2024-12-02T21:15:20,828 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8a7bf3289fb638014baa7fb12fa0d82c in 303ms, sequenceid=65, compaction requested=false 2024-12-02T21:15:20,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:20,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:20,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-02T21:15:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-02T21:15:20,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-02T21:15:20,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 784 msec 2024-12-02T21:15:20,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 799 msec 2024-12-02T21:15:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:20,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:15:20,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:20,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:20,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:20,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:20,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:20,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:20,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/dad235acbd0245979f3d7e43db8da1ef is 50, key is test_row_0/A:col10/1733174120572/Put/seqid=0 2024-12-02T21:15:20,955 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/e7220d0952e7483685da62dd17d206e1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e7220d0952e7483685da62dd17d206e1 2024-12-02T21:15:20,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174180947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174180952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174180958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174180962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174180962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:20,977 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into e7220d0952e7483685da62dd17d206e1(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:20,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741852_1028 (size=12001) 2024-12-02T21:15:20,977 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:20,977 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=13, startTime=1733174120451; duration=0sec 2024-12-02T21:15:20,977 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:20,977 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:20,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/dad235acbd0245979f3d7e43db8da1ef 2024-12-02T21:15:21,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1467ca13c0654dc7b2225a1fa01be164 is 50, key is test_row_0/B:col10/1733174120572/Put/seqid=0 2024-12-02T21:15:21,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741853_1029 (size=12001) 2024-12-02T21:15:21,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1467ca13c0654dc7b2225a1fa01be164 2024-12-02T21:15:21,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/e260c8c6fdc94272826f438100a55ec7 is 50, key is test_row_0/C:col10/1733174120572/Put/seqid=0 2024-12-02T21:15:21,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174181069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174181070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741854_1030 (size=12001) 2024-12-02T21:15:21,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/e260c8c6fdc94272826f438100a55ec7 2024-12-02T21:15:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174181080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174181080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174181081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/dad235acbd0245979f3d7e43db8da1ef as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad235acbd0245979f3d7e43db8da1ef 2024-12-02T21:15:21,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad235acbd0245979f3d7e43db8da1ef, entries=150, sequenceid=81, filesize=11.7 K 2024-12-02T21:15:21,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1467ca13c0654dc7b2225a1fa01be164 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1467ca13c0654dc7b2225a1fa01be164 2024-12-02T21:15:21,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1467ca13c0654dc7b2225a1fa01be164, entries=150, sequenceid=81, filesize=11.7 K 2024-12-02T21:15:21,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/e260c8c6fdc94272826f438100a55ec7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e260c8c6fdc94272826f438100a55ec7 2024-12-02T21:15:21,144 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/d84ab05e9b1149639de6f250d134ffee as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d84ab05e9b1149639de6f250d134ffee 2024-12-02T21:15:21,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e260c8c6fdc94272826f438100a55ec7, entries=150, sequenceid=81, filesize=11.7 K 2024-12-02T21:15:21,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8a7bf3289fb638014baa7fb12fa0d82c in 236ms, sequenceid=81, compaction requested=true 2024-12-02T21:15:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T21:15:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T21:15:21,152 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-12-02T21:15:21,154 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T21:15:21,154 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T21:15:21,154 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. because compaction request was cancelled 2024-12-02T21:15:21,154 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:21,155 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:21,157 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:21,157 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:21,157 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,157 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/275b560ad33542f184c9fbc850676f5c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3b53270d8eaa40538ae841818e62f871, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad235acbd0245979f3d7e43db8da1ef] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=35.3 K 2024-12-02T21:15:21,158 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into d84ab05e9b1149639de6f250d134ffee(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:21,158 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 275b560ad33542f184c9fbc850676f5c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174119452 2024-12-02T21:15:21,158 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:21,158 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=13, startTime=1733174120451; duration=0sec 2024-12-02T21:15:21,159 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b53270d8eaa40538ae841818e62f871, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733174119865 2024-12-02T21:15:21,159 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:21,159 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:21,160 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:21,160 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting dad235acbd0245979f3d7e43db8da1ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733174120558 2024-12-02T21:15:21,163 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:21,163 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:21,163 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,164 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e7220d0952e7483685da62dd17d206e1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/d578fd2204d04a16a997ce5ff44f8780, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1467ca13c0654dc7b2225a1fa01be164] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=35.3 K 2024-12-02T21:15:21,165 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7220d0952e7483685da62dd17d206e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174119452 2024-12-02T21:15:21,166 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting d578fd2204d04a16a997ce5ff44f8780, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733174119865 2024-12-02T21:15:21,167 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1467ca13c0654dc7b2225a1fa01be164, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733174120558 2024-12-02T21:15:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T21:15:21,180 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:21,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-02T21:15:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T21:15:21,185 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:21,187 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:21,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:21,192 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:21,192 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/91be804d3b34476496face96c08ff2d9 is 50, key is test_row_0/A:col10/1733174120572/Put/seqid=0 2024-12-02T21:15:21,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741855_1031 (size=12207) 2024-12-02T21:15:21,209 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:21,210 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ecca1c68c2114ba2b891df27a179dfb0 is 50, key is test_row_0/B:col10/1733174120572/Put/seqid=0 2024-12-02T21:15:21,229 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/91be804d3b34476496face96c08ff2d9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/91be804d3b34476496face96c08ff2d9 2024-12-02T21:15:21,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741856_1032 (size=12207) 2024-12-02T21:15:21,241 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into 91be804d3b34476496face96c08ff2d9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:21,242 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:21,242 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=13, startTime=1733174121152; duration=0sec 2024-12-02T21:15:21,242 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:21,242 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:21,257 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ecca1c68c2114ba2b891df27a179dfb0 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ecca1c68c2114ba2b891df27a179dfb0 2024-12-02T21:15:21,271 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into ecca1c68c2114ba2b891df27a179dfb0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:21,271 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:21,271 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=13, startTime=1733174121152; duration=0sec 2024-12-02T21:15:21,273 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:21,274 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:21,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T21:15:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T21:15:21,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:21,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:21,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:21,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:21,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:21,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:21,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/28f683fa46f643ab84a4001f3de6a1c6 is 50, key is test_row_0/A:col10/1733174120958/Put/seqid=0 2024-12-02T21:15:21,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741857_1033 (size=14341) 2024-12-02T21:15:21,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174181316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174181317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174181317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174181328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174181328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T21:15:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:21,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174181439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174181439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174181439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174181439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174181440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T21:15:21,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:21,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T21:15:21,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T21:15:21,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:21,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174181651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174181650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174181651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174181660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174181660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/28f683fa46f643ab84a4001f3de6a1c6 2024-12-02T21:15:21,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/c731b561644847bebbe6957cb5a0f01d is 50, key is test_row_0/B:col10/1733174120958/Put/seqid=0 2024-12-02T21:15:21,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741858_1034 (size=12001) 2024-12-02T21:15:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T21:15:21,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T21:15:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,882 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T21:15:21,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T21:15:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:21,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:21,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174181970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174181970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174181971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174181971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174181971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T21:15:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:22,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:22,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:22,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:22,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:22,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/c731b561644847bebbe6957cb5a0f01d 2024-12-02T21:15:22,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/872d788bb3a44a84a6c4671663f4558b is 50, key is test_row_0/C:col10/1733174120958/Put/seqid=0 2024-12-02T21:15:22,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T21:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:22,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:22,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:22,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741859_1035 (size=12001) 2024-12-02T21:15:22,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/872d788bb3a44a84a6c4671663f4558b 2024-12-02T21:15:22,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/28f683fa46f643ab84a4001f3de6a1c6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/28f683fa46f643ab84a4001f3de6a1c6 2024-12-02T21:15:22,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/28f683fa46f643ab84a4001f3de6a1c6, entries=200, sequenceid=104, filesize=14.0 K 2024-12-02T21:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T21:15:22,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/c731b561644847bebbe6957cb5a0f01d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/c731b561644847bebbe6957cb5a0f01d 2024-12-02T21:15:22,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/c731b561644847bebbe6957cb5a0f01d, entries=150, sequenceid=104, filesize=11.7 K 2024-12-02T21:15:22,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/872d788bb3a44a84a6c4671663f4558b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/872d788bb3a44a84a6c4671663f4558b 2024-12-02T21:15:22,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/872d788bb3a44a84a6c4671663f4558b, entries=150, sequenceid=104, filesize=11.7 K 2024-12-02T21:15:22,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 8a7bf3289fb638014baa7fb12fa0d82c in 1068ms, sequenceid=104, compaction requested=true 2024-12-02T21:15:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:22,353 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T21:15:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:22,353 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T21:15:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:22,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:22,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:22,355 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T21:15:22,355 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T21:15:22,355 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. because compaction request was cancelled 2024-12-02T21:15:22,355 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:22,355 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:22,355 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T21:15:22,355 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T21:15:22,355 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. because compaction request was cancelled 2024-12-02T21:15:22,356 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:22,357 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:22,357 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:22,358 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:22,358 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d84ab05e9b1149639de6f250d134ffee, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3475bfa9db064ec48375e48f61496bfb, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e260c8c6fdc94272826f438100a55ec7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/872d788bb3a44a84a6c4671663f4558b] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=47.0 K 2024-12-02T21:15:22,359 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting d84ab05e9b1149639de6f250d134ffee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174119452 2024-12-02T21:15:22,359 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 3475bfa9db064ec48375e48f61496bfb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733174119865 2024-12-02T21:15:22,360 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting e260c8c6fdc94272826f438100a55ec7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733174120558 2024-12-02T21:15:22,361 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 872d788bb3a44a84a6c4671663f4558b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733174120946 2024-12-02T21:15:22,387 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:22,388 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/b3b8a3776caf404888c44c3efa02b266 is 50, key is test_row_0/C:col10/1733174120958/Put/seqid=0 2024-12-02T21:15:22,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741860_1036 (size=12241) 2024-12-02T21:15:22,424 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/b3b8a3776caf404888c44c3efa02b266 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b3b8a3776caf404888c44c3efa02b266 2024-12-02T21:15:22,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T21:15:22,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:22,438 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:15:22,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:22,443 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into b3b8a3776caf404888c44c3efa02b266(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:22,447 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:22,447 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=12, startTime=1733174122354; duration=0sec 2024-12-02T21:15:22,448 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:22,448 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/3805926d62c54abd972b8a1f20834fba is 50, key is test_row_0/A:col10/1733174121311/Put/seqid=0 2024-12-02T21:15:22,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741861_1037 (size=12001) 2024-12-02T21:15:22,475 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/3805926d62c54abd972b8a1f20834fba 2024-12-02T21:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:22,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:22,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ec402734ad904a42bc02c298cc3f61f7 is 50, key is test_row_0/B:col10/1733174121311/Put/seqid=0 2024-12-02T21:15:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174182514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174182516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174182519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174182528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174182528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741862_1038 (size=12001) 2024-12-02T21:15:22,539 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ec402734ad904a42bc02c298cc3f61f7 2024-12-02T21:15:22,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1614254582cc49219a8d064bf0546267 is 50, key is test_row_0/C:col10/1733174121311/Put/seqid=0 2024-12-02T21:15:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741863_1039 (size=12001) 2024-12-02T21:15:22,603 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1614254582cc49219a8d064bf0546267 2024-12-02T21:15:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/3805926d62c54abd972b8a1f20834fba as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3805926d62c54abd972b8a1f20834fba 2024-12-02T21:15:22,632 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3805926d62c54abd972b8a1f20834fba, entries=150, sequenceid=119, filesize=11.7 K 2024-12-02T21:15:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ec402734ad904a42bc02c298cc3f61f7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ec402734ad904a42bc02c298cc3f61f7 2024-12-02T21:15:22,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174182639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174182639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174182639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174182639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174182639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,652 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ec402734ad904a42bc02c298cc3f61f7, entries=150, sequenceid=119, filesize=11.7 K 2024-12-02T21:15:22,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1614254582cc49219a8d064bf0546267 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1614254582cc49219a8d064bf0546267 2024-12-02T21:15:22,668 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1614254582cc49219a8d064bf0546267, entries=150, sequenceid=119, filesize=11.7 K 2024-12-02T21:15:22,670 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 8a7bf3289fb638014baa7fb12fa0d82c in 232ms, sequenceid=119, compaction requested=true 2024-12-02T21:15:22,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:22,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:22,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-02T21:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-02T21:15:22,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-02T21:15:22,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4860 sec 2024-12-02T21:15:22,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4990 sec 2024-12-02T21:15:22,685 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T21:15:22,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-12-02T21:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:22,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:22,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:22,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:22,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:22,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:22,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:22,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/bd2c375b21e74e88b7f8165f49dcd106 is 50, key is test_row_0/A:col10/1733174122854/Put/seqid=0 2024-12-02T21:15:22,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174182877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174182880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174182883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174182885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174182885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741864_1040 (size=16931) 2024-12-02T21:15:22,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/bd2c375b21e74e88b7f8165f49dcd106 2024-12-02T21:15:22,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/e560f79de93a4d9b8e8515036cb57e34 is 50, key is test_row_0/B:col10/1733174122854/Put/seqid=0 2024-12-02T21:15:22,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741865_1041 (size=12151) 2024-12-02T21:15:22,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/e560f79de93a4d9b8e8515036cb57e34 2024-12-02T21:15:22,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/406cf57995d045939a40b4a9cf7ebf44 is 50, key is test_row_0/C:col10/1733174122854/Put/seqid=0 2024-12-02T21:15:22,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174182989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:22,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174182989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741866_1042 (size=12151) 2024-12-02T21:15:23,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/406cf57995d045939a40b4a9cf7ebf44 2024-12-02T21:15:23,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174182999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174182999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174182999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/bd2c375b21e74e88b7f8165f49dcd106 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bd2c375b21e74e88b7f8165f49dcd106 2024-12-02T21:15:23,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bd2c375b21e74e88b7f8165f49dcd106, entries=250, sequenceid=141, filesize=16.5 K 2024-12-02T21:15:23,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/e560f79de93a4d9b8e8515036cb57e34 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e560f79de93a4d9b8e8515036cb57e34 2024-12-02T21:15:23,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e560f79de93a4d9b8e8515036cb57e34, entries=150, sequenceid=141, filesize=11.9 K 2024-12-02T21:15:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/406cf57995d045939a40b4a9cf7ebf44 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/406cf57995d045939a40b4a9cf7ebf44 2024-12-02T21:15:23,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/406cf57995d045939a40b4a9cf7ebf44, entries=150, sequenceid=141, filesize=11.9 K 2024-12-02T21:15:23,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 8a7bf3289fb638014baa7fb12fa0d82c in 203ms, sequenceid=141, compaction requested=true 2024-12-02T21:15:23,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:23,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:23,061 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:23,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:23,061 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:23,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:23,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:23,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:23,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:23,064 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:23,064 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:23,064 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:23,064 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:23,064 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:23,064 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:23,064 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/91be804d3b34476496face96c08ff2d9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/28f683fa46f643ab84a4001f3de6a1c6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3805926d62c54abd972b8a1f20834fba, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bd2c375b21e74e88b7f8165f49dcd106] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=54.2 K 2024-12-02T21:15:23,064 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ecca1c68c2114ba2b891df27a179dfb0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/c731b561644847bebbe6957cb5a0f01d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ec402734ad904a42bc02c298cc3f61f7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e560f79de93a4d9b8e8515036cb57e34] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=47.2 K 2024-12-02T21:15:23,065 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91be804d3b34476496face96c08ff2d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733174120558 2024-12-02T21:15:23,065 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ecca1c68c2114ba2b891df27a179dfb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733174120558 2024-12-02T21:15:23,065 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28f683fa46f643ab84a4001f3de6a1c6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733174120946 2024-12-02T21:15:23,065 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting c731b561644847bebbe6957cb5a0f01d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733174120946 2024-12-02T21:15:23,066 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3805926d62c54abd972b8a1f20834fba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733174121311 2024-12-02T21:15:23,066 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ec402734ad904a42bc02c298cc3f61f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733174121311 2024-12-02T21:15:23,066 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd2c375b21e74e88b7f8165f49dcd106, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733174122522 2024-12-02T21:15:23,066 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting e560f79de93a4d9b8e8515036cb57e34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733174122522 2024-12-02T21:15:23,103 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:23,103 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:23,107 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/1b632f74e3784f40ac52d1a4c3820db0 is 50, key is test_row_0/A:col10/1733174122854/Put/seqid=0 2024-12-02T21:15:23,108 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/7010577b378b43deade32bae95ba9f50 is 50, key is test_row_0/B:col10/1733174122854/Put/seqid=0 2024-12-02T21:15:23,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741867_1043 (size=12493) 2024-12-02T21:15:23,133 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/1b632f74e3784f40ac52d1a4c3820db0 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1b632f74e3784f40ac52d1a4c3820db0 2024-12-02T21:15:23,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741868_1044 (size=12493) 2024-12-02T21:15:23,148 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into 1b632f74e3784f40ac52d1a4c3820db0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:23,148 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:23,148 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=12, startTime=1733174123061; duration=0sec 2024-12-02T21:15:23,149 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:23,149 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:23,149 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:23,151 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:23,151 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:23,151 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:23,152 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b3b8a3776caf404888c44c3efa02b266, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1614254582cc49219a8d064bf0546267, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/406cf57995d045939a40b4a9cf7ebf44] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=35.5 K 2024-12-02T21:15:23,152 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3b8a3776caf404888c44c3efa02b266, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733174120946 2024-12-02T21:15:23,153 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1614254582cc49219a8d064bf0546267, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733174121311 2024-12-02T21:15:23,153 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 406cf57995d045939a40b4a9cf7ebf44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733174122522 2024-12-02T21:15:23,179 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:23,180 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/5ccc547ca4e94a60b5eac0e55eb2f998 is 50, key is test_row_0/C:col10/1733174122854/Put/seqid=0 2024-12-02T21:15:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:23,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T21:15:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:23,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:23,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:23,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:23,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741869_1045 (size=12493) 2024-12-02T21:15:23,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/8b693e8a80204269b7127fc6985e609e is 50, key is test_row_0/A:col10/1733174123203/Put/seqid=0 2024-12-02T21:15:23,226 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/5ccc547ca4e94a60b5eac0e55eb2f998 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/5ccc547ca4e94a60b5eac0e55eb2f998 2024-12-02T21:15:23,239 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into 5ccc547ca4e94a60b5eac0e55eb2f998(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:23,239 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:23,239 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=13, startTime=1733174123063; duration=0sec 2024-12-02T21:15:23,239 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:23,240 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:23,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741870_1046 (size=9757) 2024-12-02T21:15:23,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/8b693e8a80204269b7127fc6985e609e 2024-12-02T21:15:23,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174183260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174183260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/5daff2de87ea486b8cb1ac93243a1aad is 50, key is test_row_0/B:col10/1733174123203/Put/seqid=0 2024-12-02T21:15:23,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174183262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174183266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174183272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741871_1047 (size=9757) 2024-12-02T21:15:23,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/5daff2de87ea486b8cb1ac93243a1aad 2024-12-02T21:15:23,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/c80384a7352f45c58bcaca88fd26c78a is 50, key is test_row_0/C:col10/1733174123203/Put/seqid=0 2024-12-02T21:15:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T21:15:23,331 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:23,332 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-02T21:15:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T21:15:23,335 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:23,336 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:23,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:23,357 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T21:15:23,358 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T21:15:23,360 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T21:15:23,360 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T21:15:23,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741872_1048 (size=9757) 2024-12-02T21:15:23,361 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T21:15:23,361 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T21:15:23,362 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T21:15:23,362 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T21:15:23,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/c80384a7352f45c58bcaca88fd26c78a 2024-12-02T21:15:23,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/8b693e8a80204269b7127fc6985e609e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8b693e8a80204269b7127fc6985e609e 2024-12-02T21:15:23,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8b693e8a80204269b7127fc6985e609e, entries=100, sequenceid=154, filesize=9.5 K 2024-12-02T21:15:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174183390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174183390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174183391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174183391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174183391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/5daff2de87ea486b8cb1ac93243a1aad as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5daff2de87ea486b8cb1ac93243a1aad 2024-12-02T21:15:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5daff2de87ea486b8cb1ac93243a1aad, entries=100, sequenceid=154, filesize=9.5 K 2024-12-02T21:15:23,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/c80384a7352f45c58bcaca88fd26c78a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c80384a7352f45c58bcaca88fd26c78a 2024-12-02T21:15:23,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c80384a7352f45c58bcaca88fd26c78a, entries=100, sequenceid=154, filesize=9.5 K 2024-12-02T21:15:23,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8a7bf3289fb638014baa7fb12fa0d82c in 210ms, sequenceid=154, compaction requested=false 2024-12-02T21:15:23,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T21:15:23,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T21:15:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:23,494 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T21:15:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/85bbd861cece456aacb0fdcd76c2faa9 is 50, key is test_row_0/A:col10/1733174123262/Put/seqid=0 2024-12-02T21:15:23,546 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/7010577b378b43deade32bae95ba9f50 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7010577b378b43deade32bae95ba9f50 2024-12-02T21:15:23,557 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into 7010577b378b43deade32bae95ba9f50(size=12.2 K), total size for store is 21.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:23,557 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:23,557 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=12, startTime=1733174123061; duration=0sec 2024-12-02T21:15:23,558 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:23,558 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:23,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741873_1049 (size=12151) 2024-12-02T21:15:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:23,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:23,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174183615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174183616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174183617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174183620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174183622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T21:15:23,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174183729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174183729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174183729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174183729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174183730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174183939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174183939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174183940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174183940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174183940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:23,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T21:15:23,960 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/85bbd861cece456aacb0fdcd76c2faa9 2024-12-02T21:15:23,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/5f42840e0e1f4447ac20d59321cde061 is 50, key is test_row_0/B:col10/1733174123262/Put/seqid=0 2024-12-02T21:15:23,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741874_1050 (size=12151) 2024-12-02T21:15:24,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174184249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174184249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174184250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174184250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174184251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,388 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/5f42840e0e1f4447ac20d59321cde061 2024-12-02T21:15:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/526225d4284a462e83b1fe2755614952 is 50, key is test_row_0/C:col10/1733174123262/Put/seqid=0 2024-12-02T21:15:24,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741875_1051 (size=12151) 2024-12-02T21:15:24,438 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/526225d4284a462e83b1fe2755614952 2024-12-02T21:15:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/85bbd861cece456aacb0fdcd76c2faa9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/85bbd861cece456aacb0fdcd76c2faa9 2024-12-02T21:15:24,454 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/85bbd861cece456aacb0fdcd76c2faa9, entries=150, sequenceid=175, filesize=11.9 K 2024-12-02T21:15:24,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/5f42840e0e1f4447ac20d59321cde061 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5f42840e0e1f4447ac20d59321cde061 2024-12-02T21:15:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T21:15:24,471 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5f42840e0e1f4447ac20d59321cde061, entries=150, sequenceid=175, filesize=11.9 K 2024-12-02T21:15:24,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/526225d4284a462e83b1fe2755614952 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/526225d4284a462e83b1fe2755614952 2024-12-02T21:15:24,486 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/526225d4284a462e83b1fe2755614952, entries=150, sequenceid=175, filesize=11.9 K 2024-12-02T21:15:24,490 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8a7bf3289fb638014baa7fb12fa0d82c in 995ms, sequenceid=175, compaction requested=true 2024-12-02T21:15:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-02T21:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-02T21:15:24,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T21:15:24,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1550 sec 2024-12-02T21:15:24,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1640 sec 2024-12-02T21:15:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:24,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:15:24,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:24,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:24,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:24,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:24,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:24,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:24,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/1f457b1c30ee4e779a4604428c99050e is 50, key is test_row_0/A:col10/1733174124761/Put/seqid=0 2024-12-02T21:15:24,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741876_1052 (size=12151) 2024-12-02T21:15:24,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174184781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174184789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174184791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174184791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174184792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174184899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174184899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174184899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174184900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:24,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174184900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174185109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174185109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174185109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174185109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174185109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/1f457b1c30ee4e779a4604428c99050e 2024-12-02T21:15:25,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ef47922a03e64a14b06477db76767250 is 50, key is test_row_0/B:col10/1733174124761/Put/seqid=0 2024-12-02T21:15:25,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741877_1053 (size=12151) 2024-12-02T21:15:25,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ef47922a03e64a14b06477db76767250 2024-12-02T21:15:25,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1f93b847bb4c45ff8dc54f7f7c7405b6 is 50, key is test_row_0/C:col10/1733174124761/Put/seqid=0 2024-12-02T21:15:25,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741878_1054 (size=12151) 2024-12-02T21:15:25,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1f93b847bb4c45ff8dc54f7f7c7405b6 2024-12-02T21:15:25,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/1f457b1c30ee4e779a4604428c99050e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1f457b1c30ee4e779a4604428c99050e 2024-12-02T21:15:25,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1f457b1c30ee4e779a4604428c99050e, entries=150, sequenceid=192, filesize=11.9 K 2024-12-02T21:15:25,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ef47922a03e64a14b06477db76767250 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef47922a03e64a14b06477db76767250 2024-12-02T21:15:25,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef47922a03e64a14b06477db76767250, entries=150, sequenceid=192, filesize=11.9 K 2024-12-02T21:15:25,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1f93b847bb4c45ff8dc54f7f7c7405b6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1f93b847bb4c45ff8dc54f7f7c7405b6 2024-12-02T21:15:25,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1f93b847bb4c45ff8dc54f7f7c7405b6, entries=150, sequenceid=192, filesize=11.9 K 2024-12-02T21:15:25,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for 8a7bf3289fb638014baa7fb12fa0d82c in 566ms, sequenceid=192, compaction requested=true 2024-12-02T21:15:25,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:25,328 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:25,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:25,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T21:15:25,329 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:25,330 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:25,330 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:25,330 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:25,331 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1b632f74e3784f40ac52d1a4c3820db0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8b693e8a80204269b7127fc6985e609e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/85bbd861cece456aacb0fdcd76c2faa9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1f457b1c30ee4e779a4604428c99050e] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=45.5 K 2024-12-02T21:15:25,331 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:25,331 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:25,331 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:25,331 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7010577b378b43deade32bae95ba9f50, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5daff2de87ea486b8cb1ac93243a1aad, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5f42840e0e1f4447ac20d59321cde061, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef47922a03e64a14b06477db76767250] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=45.5 K 2024-12-02T21:15:25,332 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b632f74e3784f40ac52d1a4c3820db0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733174122522 2024-12-02T21:15:25,332 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7010577b378b43deade32bae95ba9f50, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733174122522 2024-12-02T21:15:25,332 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b693e8a80204269b7127fc6985e609e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733174123203 2024-12-02T21:15:25,333 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5daff2de87ea486b8cb1ac93243a1aad, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733174123203 2024-12-02T21:15:25,333 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85bbd861cece456aacb0fdcd76c2faa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733174123250 2024-12-02T21:15:25,333 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f42840e0e1f4447ac20d59321cde061, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733174123250 2024-12-02T21:15:25,333 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f457b1c30ee4e779a4604428c99050e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733174123614 2024-12-02T21:15:25,334 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ef47922a03e64a14b06477db76767250, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733174123614 2024-12-02T21:15:25,351 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:25,351 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/70c3f3e853904921a89e0d0d4405fac2 is 50, key is test_row_0/A:col10/1733174124761/Put/seqid=0 2024-12-02T21:15:25,361 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:25,362 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/8468001cbed04e78a845a7bb0821fd65 is 50, key is test_row_0/B:col10/1733174124761/Put/seqid=0 2024-12-02T21:15:25,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741879_1055 (size=12629) 2024-12-02T21:15:25,378 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/70c3f3e853904921a89e0d0d4405fac2 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/70c3f3e853904921a89e0d0d4405fac2 2024-12-02T21:15:25,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741880_1056 (size=12629) 2024-12-02T21:15:25,389 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into 70c3f3e853904921a89e0d0d4405fac2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:25,389 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:25,389 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=12, startTime=1733174125328; duration=0sec 2024-12-02T21:15:25,390 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:25,390 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:25,391 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:25,394 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:25,394 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:25,394 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:25,394 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/5ccc547ca4e94a60b5eac0e55eb2f998, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c80384a7352f45c58bcaca88fd26c78a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/526225d4284a462e83b1fe2755614952, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1f93b847bb4c45ff8dc54f7f7c7405b6] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=45.5 K 2024-12-02T21:15:25,395 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ccc547ca4e94a60b5eac0e55eb2f998, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733174122522 2024-12-02T21:15:25,396 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting c80384a7352f45c58bcaca88fd26c78a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733174123203 2024-12-02T21:15:25,399 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/8468001cbed04e78a845a7bb0821fd65 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/8468001cbed04e78a845a7bb0821fd65 2024-12-02T21:15:25,403 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 526225d4284a462e83b1fe2755614952, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733174123250 2024-12-02T21:15:25,404 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f93b847bb4c45ff8dc54f7f7c7405b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733174123614 2024-12-02T21:15:25,409 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into 8468001cbed04e78a845a7bb0821fd65(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:25,409 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:25,409 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=12, startTime=1733174125329; duration=0sec 2024-12-02T21:15:25,409 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:25,409 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:25,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T21:15:25,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:25,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:25,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:25,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:25,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:25,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:25,424 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:25,425 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/103f3165486545c198be94b07719e8b0 is 50, key is test_row_0/C:col10/1733174124761/Put/seqid=0 2024-12-02T21:15:25,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/dad7bb5ec4674b28938028d99698473b is 50, key is test_row_0/A:col10/1733174125421/Put/seqid=0 2024-12-02T21:15:25,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174185434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174185435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174185436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174185442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174185444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741881_1057 (size=12629) 2024-12-02T21:15:25,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741882_1058 (size=12151) 2024-12-02T21:15:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T21:15:25,479 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:25,479 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-02T21:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T21:15:25,485 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:25,487 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:25,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:25,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174185548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174185549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174185549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174185555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174185555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T21:15:25,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T21:15:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:25,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:25,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174185759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174185760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174185770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174185770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174185770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:25,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T21:15:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:25,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T21:15:25,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/dad7bb5ec4674b28938028d99698473b 2024-12-02T21:15:25,874 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/103f3165486545c198be94b07719e8b0 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/103f3165486545c198be94b07719e8b0 2024-12-02T21:15:25,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/341ce53532184d019ed4b73491e1ab19 is 50, key is test_row_0/B:col10/1733174125421/Put/seqid=0 2024-12-02T21:15:25,887 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into 103f3165486545c198be94b07719e8b0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:25,887 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:25,887 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=12, startTime=1733174125329; duration=0sec 2024-12-02T21:15:25,887 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:25,887 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741883_1059 (size=12151) 2024-12-02T21:15:25,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T21:15:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:25,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174186069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174186069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174186078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174186078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174186079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T21:15:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T21:15:26,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T21:15:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:26,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/341ce53532184d019ed4b73491e1ab19 2024-12-02T21:15:26,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1d5cd51bfc634ac885c8a59beb1a9e79 is 50, key is test_row_0/C:col10/1733174125421/Put/seqid=0 2024-12-02T21:15:26,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741884_1060 (size=12151) 2024-12-02T21:15:26,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T21:15:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T21:15:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174186579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174186579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174186588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174186589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174186589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T21:15:26,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1d5cd51bfc634ac885c8a59beb1a9e79 2024-12-02T21:15:26,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T21:15:26,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:26,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:26,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/dad7bb5ec4674b28938028d99698473b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad7bb5ec4674b28938028d99698473b 2024-12-02T21:15:26,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad7bb5ec4674b28938028d99698473b, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T21:15:26,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/341ce53532184d019ed4b73491e1ab19 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/341ce53532184d019ed4b73491e1ab19 2024-12-02T21:15:26,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/341ce53532184d019ed4b73491e1ab19, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T21:15:26,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1d5cd51bfc634ac885c8a59beb1a9e79 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1d5cd51bfc634ac885c8a59beb1a9e79 2024-12-02T21:15:26,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1d5cd51bfc634ac885c8a59beb1a9e79, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T21:15:26,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8a7bf3289fb638014baa7fb12fa0d82c in 1377ms, sequenceid=214, compaction requested=false 2024-12-02T21:15:26,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:26,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T21:15:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:26,882 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T21:15:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:26,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:26,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:26,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:26,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/8355e1081e2f4f6f8de1fc0dcc4376e0 is 50, key is test_row_0/A:col10/1733174125438/Put/seqid=0 2024-12-02T21:15:26,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741885_1061 (size=12151) 2024-12-02T21:15:26,926 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/8355e1081e2f4f6f8de1fc0dcc4376e0 2024-12-02T21:15:26,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/5780dfd920b84f4487ad7ff72b681e63 is 50, key is test_row_0/B:col10/1733174125438/Put/seqid=0 2024-12-02T21:15:26,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741886_1062 (size=12151) 2024-12-02T21:15:27,355 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/5780dfd920b84f4487ad7ff72b681e63 2024-12-02T21:15:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/08425a8f92d0431caf714f1013641b78 is 50, key is test_row_0/C:col10/1733174125438/Put/seqid=0 2024-12-02T21:15:27,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741887_1063 (size=12151) 2024-12-02T21:15:27,417 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/08425a8f92d0431caf714f1013641b78 2024-12-02T21:15:27,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/8355e1081e2f4f6f8de1fc0dcc4376e0 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8355e1081e2f4f6f8de1fc0dcc4376e0 2024-12-02T21:15:27,445 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8355e1081e2f4f6f8de1fc0dcc4376e0, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T21:15:27,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/5780dfd920b84f4487ad7ff72b681e63 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5780dfd920b84f4487ad7ff72b681e63 2024-12-02T21:15:27,459 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5780dfd920b84f4487ad7ff72b681e63, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T21:15:27,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/08425a8f92d0431caf714f1013641b78 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08425a8f92d0431caf714f1013641b78 2024-12-02T21:15:27,477 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08425a8f92d0431caf714f1013641b78, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T21:15:27,484 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 8a7bf3289fb638014baa7fb12fa0d82c in 598ms, sequenceid=228, compaction requested=true 2024-12-02T21:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-02T21:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-02T21:15:27,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-02T21:15:27,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0020 sec 2024-12-02T21:15:27,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.0290 sec 2024-12-02T21:15:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:27,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:15:27,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:27,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:27,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:27,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:27,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:27,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:27,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/09b90d372da74587b8b478217bf2c585 is 50, key is test_row_0/A:col10/1733174127613/Put/seqid=0 2024-12-02T21:15:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T21:15:27,630 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:27,630 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-02T21:15:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T21:15:27,635 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:27,637 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:27,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:27,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741888_1064 (size=12151) 2024-12-02T21:15:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174187651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174187653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174187655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174187657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174187659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T21:15:27,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174187769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174187769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174187769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174187770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174187770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T21:15:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:27,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:27,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T21:15:27,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:27,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:27,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:27,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:27,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T21:15:27,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174187980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174187980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174187980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174187980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174187981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/09b90d372da74587b8b478217bf2c585 2024-12-02T21:15:28,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/209235fc0c114ae5957a900cd5c69d5e is 50, key is test_row_0/B:col10/1733174127613/Put/seqid=0 2024-12-02T21:15:28,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741889_1065 (size=12151) 2024-12-02T21:15:28,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/209235fc0c114ae5957a900cd5c69d5e 2024-12-02T21:15:28,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/37906e1ab4ff4bba912aa868bdd8969c is 50, key is test_row_0/C:col10/1733174127613/Put/seqid=0 2024-12-02T21:15:28,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T21:15:28,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:28,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741890_1066 (size=12151) 2024-12-02T21:15:28,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T21:15:28,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:28,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:28,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T21:15:28,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174188288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174188289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174188289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174188289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174188289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T21:15:28,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:28,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:28,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:28,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/37906e1ab4ff4bba912aa868bdd8969c 2024-12-02T21:15:28,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/09b90d372da74587b8b478217bf2c585 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/09b90d372da74587b8b478217bf2c585 2024-12-02T21:15:28,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/09b90d372da74587b8b478217bf2c585, entries=150, sequenceid=240, filesize=11.9 K 2024-12-02T21:15:28,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/209235fc0c114ae5957a900cd5c69d5e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/209235fc0c114ae5957a900cd5c69d5e 2024-12-02T21:15:28,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/209235fc0c114ae5957a900cd5c69d5e, entries=150, sequenceid=240, filesize=11.9 K 2024-12-02T21:15:28,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/37906e1ab4ff4bba912aa868bdd8969c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/37906e1ab4ff4bba912aa868bdd8969c 2024-12-02T21:15:28,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/37906e1ab4ff4bba912aa868bdd8969c, entries=150, sequenceid=240, filesize=11.9 K 2024-12-02T21:15:28,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8a7bf3289fb638014baa7fb12fa0d82c in 926ms, sequenceid=240, compaction requested=true 2024-12-02T21:15:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:28,544 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:28,544 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:28,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:28,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:28,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:28,545 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:28,545 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:28,545 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:28,546 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,546 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:28,546 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,546 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/70c3f3e853904921a89e0d0d4405fac2, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad7bb5ec4674b28938028d99698473b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8355e1081e2f4f6f8de1fc0dcc4376e0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/09b90d372da74587b8b478217bf2c585] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=47.9 K 2024-12-02T21:15:28,546 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/8468001cbed04e78a845a7bb0821fd65, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/341ce53532184d019ed4b73491e1ab19, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5780dfd920b84f4487ad7ff72b681e63, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/209235fc0c114ae5957a900cd5c69d5e] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=47.9 K 2024-12-02T21:15:28,546 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c3f3e853904921a89e0d0d4405fac2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733174123614 2024-12-02T21:15:28,546 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 8468001cbed04e78a845a7bb0821fd65, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733174123614 2024-12-02T21:15:28,547 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 341ce53532184d019ed4b73491e1ab19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733174124786 2024-12-02T21:15:28,547 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting dad7bb5ec4674b28938028d99698473b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733174124786 2024-12-02T21:15:28,547 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5780dfd920b84f4487ad7ff72b681e63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733174125424 2024-12-02T21:15:28,547 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8355e1081e2f4f6f8de1fc0dcc4376e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733174125424 2024-12-02T21:15:28,547 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09b90d372da74587b8b478217bf2c585, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733174127611 2024-12-02T21:15:28,547 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 209235fc0c114ae5957a900cd5c69d5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733174127611 2024-12-02T21:15:28,560 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:28,560 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:28,561 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/37bf2f39c2004ef3936274905c810c87 is 50, key is test_row_0/A:col10/1733174127613/Put/seqid=0 2024-12-02T21:15:28,562 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/57f7317e8dc84c82b1f97495899ceda5 is 50, key is test_row_0/B:col10/1733174127613/Put/seqid=0 2024-12-02T21:15:28,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T21:15:28,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,565 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T21:15:28,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:28,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:28,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:28,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:28,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:28,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/f2ebba3c0a6f4f968960008328699200 is 50, key is test_row_0/A:col10/1733174127640/Put/seqid=0 2024-12-02T21:15:28,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741891_1067 (size=12765) 2024-12-02T21:15:28,600 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/57f7317e8dc84c82b1f97495899ceda5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/57f7317e8dc84c82b1f97495899ceda5 2024-12-02T21:15:28,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741892_1068 (size=12765) 2024-12-02T21:15:28,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741893_1069 (size=12251) 2024-12-02T21:15:28,608 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into 57f7317e8dc84c82b1f97495899ceda5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:28,608 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:28,608 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=12, startTime=1733174128544; duration=0sec 2024-12-02T21:15:28,608 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:28,608 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:28,608 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:28,610 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:28,610 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:28,611 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/37bf2f39c2004ef3936274905c810c87 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/37bf2f39c2004ef3936274905c810c87 2024-12-02T21:15:28,611 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:28,612 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/103f3165486545c198be94b07719e8b0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1d5cd51bfc634ac885c8a59beb1a9e79, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08425a8f92d0431caf714f1013641b78, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/37906e1ab4ff4bba912aa868bdd8969c] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=47.9 K 2024-12-02T21:15:28,612 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 103f3165486545c198be94b07719e8b0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733174123614 2024-12-02T21:15:28,613 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d5cd51bfc634ac885c8a59beb1a9e79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733174124786 2024-12-02T21:15:28,613 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 08425a8f92d0431caf714f1013641b78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733174125424 2024-12-02T21:15:28,614 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 37906e1ab4ff4bba912aa868bdd8969c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733174127611 2024-12-02T21:15:28,624 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into 37bf2f39c2004ef3936274905c810c87(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:28,624 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:28,624 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=12, startTime=1733174128544; duration=0sec 2024-12-02T21:15:28,624 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:28,624 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:28,631 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#57 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:28,632 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/9e4f30a1093a4514be1a7ccd33b71a46 is 50, key is test_row_0/C:col10/1733174127613/Put/seqid=0 2024-12-02T21:15:28,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741894_1070 (size=12765) 2024-12-02T21:15:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T21:15:28,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174188811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174188813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174188811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174188815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174188815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174188929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174188929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174188929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174188929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174188929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,009 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/f2ebba3c0a6f4f968960008328699200 2024-12-02T21:15:29,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b8f36f1048854e52984e55a92414fe87 is 50, key is test_row_0/B:col10/1733174127640/Put/seqid=0 2024-12-02T21:15:29,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741895_1071 (size=12251) 2024-12-02T21:15:29,058 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/9e4f30a1093a4514be1a7ccd33b71a46 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/9e4f30a1093a4514be1a7ccd33b71a46 2024-12-02T21:15:29,069 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into 9e4f30a1093a4514be1a7ccd33b71a46(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:29,069 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:29,069 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=12, startTime=1733174128545; duration=0sec 2024-12-02T21:15:29,069 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:29,069 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:29,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174189139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174189140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174189140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174189140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174189139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,430 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b8f36f1048854e52984e55a92414fe87 2024-12-02T21:15:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/23d6a6a62a2743b78d12fd734aa57d7f is 50, key is test_row_0/C:col10/1733174127640/Put/seqid=0 2024-12-02T21:15:29,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174189449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174189450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741896_1072 (size=12251) 2024-12-02T21:15:29,456 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/23d6a6a62a2743b78d12fd734aa57d7f 2024-12-02T21:15:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174189459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174189459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174189459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:29,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/f2ebba3c0a6f4f968960008328699200 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/f2ebba3c0a6f4f968960008328699200 2024-12-02T21:15:29,485 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/f2ebba3c0a6f4f968960008328699200, entries=150, sequenceid=260, filesize=12.0 K 2024-12-02T21:15:29,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b8f36f1048854e52984e55a92414fe87 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b8f36f1048854e52984e55a92414fe87 2024-12-02T21:15:29,497 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b8f36f1048854e52984e55a92414fe87, entries=150, sequenceid=260, filesize=12.0 K 2024-12-02T21:15:29,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/23d6a6a62a2743b78d12fd734aa57d7f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/23d6a6a62a2743b78d12fd734aa57d7f 2024-12-02T21:15:29,514 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/23d6a6a62a2743b78d12fd734aa57d7f, entries=150, sequenceid=260, filesize=12.0 K 2024-12-02T21:15:29,516 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8a7bf3289fb638014baa7fb12fa0d82c in 951ms, sequenceid=260, compaction requested=false 2024-12-02T21:15:29,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-02T21:15:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-02T21:15:29,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-02T21:15:29,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-12-02T21:15:29,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8930 sec 2024-12-02T21:15:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T21:15:29,779 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:29,780 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-02T21:15:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T21:15:29,786 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:29,801 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:29,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T21:15:29,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T21:15:29,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:29,955 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T21:15:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:29,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/4e33ffff965f4b7484f16f338549c8b6 is 50, key is test_row_0/A:col10/1733174128809/Put/seqid=0 2024-12-02T21:15:30,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741897_1073 (size=12301) 2024-12-02T21:15:30,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174190004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174190006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174190009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174190011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174190011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T21:15:30,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174190119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174190119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174190120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174190129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174190129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174190329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174190329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174190329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174190340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174190340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,408 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/4e33ffff965f4b7484f16f338549c8b6 2024-12-02T21:15:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T21:15:30,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b67135bd136d4db1bd059e3987915910 is 50, key is test_row_0/B:col10/1733174128809/Put/seqid=0 2024-12-02T21:15:30,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741898_1074 (size=12301) 2024-12-02T21:15:30,455 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b67135bd136d4db1bd059e3987915910 2024-12-02T21:15:30,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/e3f02059ccc945858094d5729e614bfd is 50, key is test_row_0/C:col10/1733174128809/Put/seqid=0 2024-12-02T21:15:30,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741899_1075 (size=12301) 2024-12-02T21:15:30,489 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/e3f02059ccc945858094d5729e614bfd 2024-12-02T21:15:30,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/4e33ffff965f4b7484f16f338549c8b6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4e33ffff965f4b7484f16f338549c8b6 2024-12-02T21:15:30,500 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4e33ffff965f4b7484f16f338549c8b6, entries=150, sequenceid=276, filesize=12.0 K 2024-12-02T21:15:30,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b67135bd136d4db1bd059e3987915910 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b67135bd136d4db1bd059e3987915910 2024-12-02T21:15:30,508 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b67135bd136d4db1bd059e3987915910, entries=150, sequenceid=276, filesize=12.0 K 2024-12-02T21:15:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/e3f02059ccc945858094d5729e614bfd as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e3f02059ccc945858094d5729e614bfd 2024-12-02T21:15:30,520 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e3f02059ccc945858094d5729e614bfd, entries=150, sequenceid=276, filesize=12.0 K 2024-12-02T21:15:30,521 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8a7bf3289fb638014baa7fb12fa0d82c in 565ms, sequenceid=276, compaction requested=true 2024-12-02T21:15:30,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:30,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:30,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-02T21:15:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-02T21:15:30,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-02T21:15:30,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 721 msec 2024-12-02T21:15:30,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 756 msec 2024-12-02T21:15:30,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174130005, max timestamp=1733174130650Num uniques 859; 2024-12-02T21:15:30,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174130005, max timestamp=1733174130650Num uniques 859; 2024-12-02T21:15:30,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:30,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:30,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=71.56 KB heapSize=204.02 KB 2024-12-02T21:15:30,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733174130005, max timestamp=1733174130650Num uniques 859; 2024-12-02T21:15:30,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T21:15:30,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:30,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/620e9702f6214ad39506bd866980f27f is 50, key is test_row_0/A:col10/1733174130659/Put/seqid=0 2024-12-02T21:15:30,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741900_1076 (size=14741) 2024-12-02T21:15:30,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174130703, max timestamp=1733174130716Num uniques 859; 2024-12-02T21:15:30,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:30,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174130703, max timestamp=1733174130717Num uniques 859; 2024-12-02T21:15:30,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:30,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733174130703, max timestamp=1733174130716Num uniques 859; 2024-12-02T21:15:30,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T21:15:30,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174190752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174190754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733174130743, max timestamp=1733174130761Num uniques 1018; 2024-12-02T21:15:30,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733174130743, max timestamp=1733174130761Num uniques 1009; 2024-12-02T21:15:30,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-02T21:15:30,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733174130743, max timestamp=1733174130761Num uniques 1018; 2024-12-02T21:15:30,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=14558, suffix heap size=132904,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=318 2024-12-02T21:15:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174190750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-02T21:15:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174190839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174190848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174190889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174190889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174190889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T21:15:30,919 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:30,920 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-02T21:15:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T21:15:30,923 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:30,925 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:30,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:30,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174190959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174190959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T21:15:31,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T21:15:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/620e9702f6214ad39506bd866980f27f 2024-12-02T21:15:31,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174191099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174191099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174191099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1e0c64e7910a452db695de1660e25826 is 50, key is test_row_0/B:col10/1733174130659/Put/seqid=0 2024-12-02T21:15:31,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741901_1077 (size=12301) 2024-12-02T21:15:31,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174191169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174191169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T21:15:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:31,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T21:15:31,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T21:15:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174191409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174191409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174191409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174191479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174191479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:31,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:31,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:31,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1e0c64e7910a452db695de1660e25826 2024-12-02T21:15:31,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T21:15:31,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:31,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T21:15:31,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/ea90d3ac15e24839a22c6c98bf51c564 is 50, key is test_row_0/C:col10/1733174130659/Put/seqid=0 2024-12-02T21:15:31,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741902_1078 (size=12301) 2024-12-02T21:15:31,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.26 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/ea90d3ac15e24839a22c6c98bf51c564 2024-12-02T21:15:31,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/620e9702f6214ad39506bd866980f27f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/620e9702f6214ad39506bd866980f27f 2024-12-02T21:15:31,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/620e9702f6214ad39506bd866980f27f, entries=200, sequenceid=300, filesize=14.4 K 2024-12-02T21:15:31,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1e0c64e7910a452db695de1660e25826 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1e0c64e7910a452db695de1660e25826 2024-12-02T21:15:31,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1e0c64e7910a452db695de1660e25826, entries=150, sequenceid=300, filesize=12.0 K 2024-12-02T21:15:31,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/ea90d3ac15e24839a22c6c98bf51c564 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/ea90d3ac15e24839a22c6c98bf51c564 2024-12-02T21:15:31,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/ea90d3ac15e24839a22c6c98bf51c564, entries=150, sequenceid=300, filesize=12.0 K 2024-12-02T21:15:31,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.57 KB/47685, heapSize ~125.04 KB/128040, currentSize=140.89 KB/144270 for 8a7bf3289fb638014baa7fb12fa0d82c in 1014ms, sequenceid=300, compaction requested=true 2024-12-02T21:15:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:31,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:31,674 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:31,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:31,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:31,674 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:31,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:31,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:31,676 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52058 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:31,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:31,676 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:31,676 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,676 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/37bf2f39c2004ef3936274905c810c87, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/f2ebba3c0a6f4f968960008328699200, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4e33ffff965f4b7484f16f338549c8b6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/620e9702f6214ad39506bd866980f27f] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=50.8 K 2024-12-02T21:15:31,677 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49618 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:31,677 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:31,677 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,678 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/57f7317e8dc84c82b1f97495899ceda5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b8f36f1048854e52984e55a92414fe87, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b67135bd136d4db1bd059e3987915910, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1e0c64e7910a452db695de1660e25826] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=48.5 K 2024-12-02T21:15:31,678 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37bf2f39c2004ef3936274905c810c87, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733174127611 2024-12-02T21:15:31,678 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 57f7317e8dc84c82b1f97495899ceda5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733174127611 2024-12-02T21:15:31,679 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ebba3c0a6f4f968960008328699200, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733174127640 2024-12-02T21:15:31,679 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f36f1048854e52984e55a92414fe87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733174127640 2024-12-02T21:15:31,679 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e33ffff965f4b7484f16f338549c8b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733174128809 2024-12-02T21:15:31,679 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b67135bd136d4db1bd059e3987915910, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733174128809 2024-12-02T21:15:31,680 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 620e9702f6214ad39506bd866980f27f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733174130650 2024-12-02T21:15:31,680 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0c64e7910a452db695de1660e25826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733174130650 2024-12-02T21:15:31,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T21:15:31,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,697 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=140.89 KB heapSize=390 KB 2024-12-02T21:15:31,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:31,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:31,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:31,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:31,713 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:31,714 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/3df2699b21eb40c1b9baeab81ef02918 is 50, key is test_row_0/A:col10/1733174130659/Put/seqid=0 2024-12-02T21:15:31,735 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:31,736 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/7c617cd82ea147d68c25d86ca56f4717 is 50, key is test_row_0/B:col10/1733174130659/Put/seqid=0 2024-12-02T21:15:31,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/642d07b435cd4bfa9829c6f0c5b7d7fd is 50, key is test_row_0/A:col10/1733174130835/Put/seqid=0 2024-12-02T21:15:31,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741903_1079 (size=13051) 2024-12-02T21:15:31,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741904_1080 (size=13051) 2024-12-02T21:15:31,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741905_1081 (size=12301) 2024-12-02T21:15:31,781 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/642d07b435cd4bfa9829c6f0c5b7d7fd 2024-12-02T21:15:31,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/bf2d0814b00f495f86f830c5137fa911 is 50, key is test_row_0/B:col10/1733174130835/Put/seqid=0 2024-12-02T21:15:31,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741906_1082 (size=12301) 2024-12-02T21:15:31,802 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/bf2d0814b00f495f86f830c5137fa911 2024-12-02T21:15:31,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/b8871e85a30c49ad95f1d51a516eff1e is 50, key is test_row_0/C:col10/1733174130835/Put/seqid=0 2024-12-02T21:15:31,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741907_1083 (size=12301) 2024-12-02T21:15:31,861 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/b8871e85a30c49ad95f1d51a516eff1e 2024-12-02T21:15:31,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/642d07b435cd4bfa9829c6f0c5b7d7fd as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/642d07b435cd4bfa9829c6f0c5b7d7fd 2024-12-02T21:15:31,887 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/642d07b435cd4bfa9829c6f0c5b7d7fd, entries=150, sequenceid=352, filesize=12.0 K 2024-12-02T21:15:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/bf2d0814b00f495f86f830c5137fa911 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/bf2d0814b00f495f86f830c5137fa911 2024-12-02T21:15:31,901 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/bf2d0814b00f495f86f830c5137fa911, entries=150, sequenceid=352, filesize=12.0 K 2024-12-02T21:15:31,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/b8871e85a30c49ad95f1d51a516eff1e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b8871e85a30c49ad95f1d51a516eff1e 2024-12-02T21:15:31,918 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b8871e85a30c49ad95f1d51a516eff1e, entries=150, sequenceid=352, filesize=12.0 K 2024-12-02T21:15:31,919 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~389.95 KB/399312, currentSize=0 B/0 for 8a7bf3289fb638014baa7fb12fa0d82c in 222ms, sequenceid=352, compaction requested=true 2024-12-02T21:15:31,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:31,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:31,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-02T21:15:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-02T21:15:31,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-02T21:15:31,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 996 msec 2024-12-02T21:15:31,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0080 sec 2024-12-02T21:15:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:31,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:15:31,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:31,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:31,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:31,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:31,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:31,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:31,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/763f3f426bd84e0999570c0f6b69d995 is 50, key is test_row_0/A:col10/1733174131947/Put/seqid=0 2024-12-02T21:15:31,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741908_1084 (size=14741) 2024-12-02T21:15:31,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/763f3f426bd84e0999570c0f6b69d995 2024-12-02T21:15:31,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/f063c527cdea4cc0a756ba63e1d99523 is 50, key is test_row_0/B:col10/1733174131947/Put/seqid=0 2024-12-02T21:15:32,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733174131977, max timestamp=1733174131998Num uniques 859; 2024-12-02T21:15:32,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733174131977, max timestamp=1733174131998Num uniques 859; 2024-12-02T21:15:32,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733174131977, max timestamp=1733174131998Num uniques 859; 2024-12-02T21:15:32,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T21:15:32,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T21:15:32,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T21:15:32,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741909_1085 (size=12301) 2024-12-02T21:15:32,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/f063c527cdea4cc0a756ba63e1d99523 2024-12-02T21:15:32,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/e75de9370d6049aea4bfe4a4c289688a is 50, key is test_row_0/C:col10/1733174131947/Put/seqid=0 2024-12-02T21:15:32,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741910_1086 (size=12301) 2024-12-02T21:15:32,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/e75de9370d6049aea4bfe4a4c289688a 2024-12-02T21:15:32,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174192025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/763f3f426bd84e0999570c0f6b69d995 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/763f3f426bd84e0999570c0f6b69d995 2024-12-02T21:15:32,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174192028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174192027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174192030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174192032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/763f3f426bd84e0999570c0f6b69d995, entries=200, sequenceid=362, filesize=14.4 K 2024-12-02T21:15:32,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/f063c527cdea4cc0a756ba63e1d99523 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f063c527cdea4cc0a756ba63e1d99523 2024-12-02T21:15:32,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f063c527cdea4cc0a756ba63e1d99523, entries=150, sequenceid=362, filesize=12.0 K 2024-12-02T21:15:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T21:15:32,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/e75de9370d6049aea4bfe4a4c289688a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e75de9370d6049aea4bfe4a4c289688a 2024-12-02T21:15:32,059 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:32,060 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-02T21:15:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T21:15:32,066 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:32,067 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:32,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:32,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e75de9370d6049aea4bfe4a4c289688a, entries=150, sequenceid=362, filesize=12.0 K 2024-12-02T21:15:32,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 8a7bf3289fb638014baa7fb12fa0d82c in 123ms, sequenceid=362, compaction requested=true 2024-12-02T21:15:32,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:32,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:32,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T21:15:32,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:32,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T21:15:32,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:32,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T21:15:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:32,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-12-02T21:15:32,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:32,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:32,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:32,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:32,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:32,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:32,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/9b3bbe820e9e4cafbdcc5863db36aee5 is 50, key is test_row_0/A:col10/1733174132021/Put/seqid=0 2024-12-02T21:15:32,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741911_1087 (size=14741) 2024-12-02T21:15:32,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174192155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174192157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174192157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174192158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174192159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T21:15:32,181 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/3df2699b21eb40c1b9baeab81ef02918 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3df2699b21eb40c1b9baeab81ef02918 2024-12-02T21:15:32,186 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/7c617cd82ea147d68c25d86ca56f4717 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7c617cd82ea147d68c25d86ca56f4717 2024-12-02T21:15:32,188 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into 3df2699b21eb40c1b9baeab81ef02918(size=12.7 K), total size for store is 39.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:32,188 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:32,188 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=12, startTime=1733174131674; duration=0sec 2024-12-02T21:15:32,188 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T21:15:32,188 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:32,188 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:32,188 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T21:15:32,196 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74220 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T21:15:32,196 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:32,197 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into 7c617cd82ea147d68c25d86ca56f4717(size=12.7 K), total size for store is 36.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:32,197 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:32,197 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=12, startTime=1733174131674; duration=0sec 2024-12-02T21:15:32,197 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T21:15:32,197 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:32,197 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:32,198 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 6 compacting, 0 eligible, 16 blocking 2024-12-02T21:15:32,198 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T21:15:32,198 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T21:15:32,198 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. because compaction request was cancelled 2024-12-02T21:15:32,198 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:32,198 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:32,199 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,199 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/9e4f30a1093a4514be1a7ccd33b71a46, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/23d6a6a62a2743b78d12fd734aa57d7f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e3f02059ccc945858094d5729e614bfd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/ea90d3ac15e24839a22c6c98bf51c564, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b8871e85a30c49ad95f1d51a516eff1e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e75de9370d6049aea4bfe4a4c289688a] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=72.5 K 2024-12-02T21:15:32,201 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e4f30a1093a4514be1a7ccd33b71a46, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733174127611 2024-12-02T21:15:32,202 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23d6a6a62a2743b78d12fd734aa57d7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733174127640 2024-12-02T21:15:32,202 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:32,203 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:32,203 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,203 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7c617cd82ea147d68c25d86ca56f4717, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/bf2d0814b00f495f86f830c5137fa911, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f063c527cdea4cc0a756ba63e1d99523] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=36.8 K 2024-12-02T21:15:32,204 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3f02059ccc945858094d5729e614bfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733174128809 2024-12-02T21:15:32,204 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea90d3ac15e24839a22c6c98bf51c564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733174130650 2024-12-02T21:15:32,205 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c617cd82ea147d68c25d86ca56f4717, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733174130650 2024-12-02T21:15:32,205 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8871e85a30c49ad95f1d51a516eff1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733174130828 2024-12-02T21:15:32,205 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2d0814b00f495f86f830c5137fa911, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733174130828 2024-12-02T21:15:32,206 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e75de9370d6049aea4bfe4a4c289688a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733174131938 2024-12-02T21:15:32,207 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f063c527cdea4cc0a756ba63e1d99523, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733174131938 2024-12-02T21:15:32,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T21:15:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,235 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:32,236 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b479258b65d24026bfde2f27e9ee565e is 50, key is test_row_0/B:col10/1733174131947/Put/seqid=0 2024-12-02T21:15:32,248 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:32,248 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/41c4e0506b324853bd1da1095bbfe7da is 50, key is test_row_0/C:col10/1733174131947/Put/seqid=0 2024-12-02T21:15:32,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741912_1088 (size=13153) 2024-12-02T21:15:32,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741913_1089 (size=13119) 2024-12-02T21:15:32,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174192269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174192269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174192269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,273 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/41c4e0506b324853bd1da1095bbfe7da as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/41c4e0506b324853bd1da1095bbfe7da 2024-12-02T21:15:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174192271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174192271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,293 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into 41c4e0506b324853bd1da1095bbfe7da(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:32,293 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:32,293 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=10, startTime=1733174132071; duration=0sec 2024-12-02T21:15:32,294 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:32,294 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:32,294 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:32,297 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:32,298 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:32,298 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,298 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3df2699b21eb40c1b9baeab81ef02918, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/642d07b435cd4bfa9829c6f0c5b7d7fd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/763f3f426bd84e0999570c0f6b69d995] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=39.2 K 2024-12-02T21:15:32,299 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3df2699b21eb40c1b9baeab81ef02918, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733174130650 2024-12-02T21:15:32,299 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 642d07b435cd4bfa9829c6f0c5b7d7fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733174130828 2024-12-02T21:15:32,300 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 763f3f426bd84e0999570c0f6b69d995, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733174131930 2024-12-02T21:15:32,312 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:32,313 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/65ad647c97d641fdbf213ff63e9737f4 is 50, key is test_row_0/A:col10/1733174131947/Put/seqid=0 2024-12-02T21:15:32,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741914_1090 (size=13153) 2024-12-02T21:15:32,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T21:15:32,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:32,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T21:15:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174192478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174192479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174192479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174192479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174192489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T21:15:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:32,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/9b3bbe820e9e4cafbdcc5863db36aee5 2024-12-02T21:15:32,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/52d032c9682c449d9a133e6aa5a28bc9 is 50, key is test_row_0/B:col10/1733174132021/Put/seqid=0 2024-12-02T21:15:32,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741915_1091 (size=12301) 2024-12-02T21:15:32,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/52d032c9682c449d9a133e6aa5a28bc9 2024-12-02T21:15:32,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/d92e74d3f90c4f1482461fe4e05c90ac is 50, key is test_row_0/C:col10/1733174132021/Put/seqid=0 2024-12-02T21:15:32,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741916_1092 (size=12301) 2024-12-02T21:15:32,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/d92e74d3f90c4f1482461fe4e05c90ac 2024-12-02T21:15:32,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/9b3bbe820e9e4cafbdcc5863db36aee5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/9b3bbe820e9e4cafbdcc5863db36aee5 2024-12-02T21:15:32,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/9b3bbe820e9e4cafbdcc5863db36aee5, entries=200, sequenceid=397, filesize=14.4 K 2024-12-02T21:15:32,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/52d032c9682c449d9a133e6aa5a28bc9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/52d032c9682c449d9a133e6aa5a28bc9 2024-12-02T21:15:32,670 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b479258b65d24026bfde2f27e9ee565e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b479258b65d24026bfde2f27e9ee565e 2024-12-02T21:15:32,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/52d032c9682c449d9a133e6aa5a28bc9, entries=150, sequenceid=397, filesize=12.0 K 2024-12-02T21:15:32,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/d92e74d3f90c4f1482461fe4e05c90ac as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d92e74d3f90c4f1482461fe4e05c90ac 2024-12-02T21:15:32,679 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into b479258b65d24026bfde2f27e9ee565e(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:32,679 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:32,679 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=13, startTime=1733174132071; duration=0sec 2024-12-02T21:15:32,680 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:32,680 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:32,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T21:15:32,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T21:15:32,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d92e74d3f90c4f1482461fe4e05c90ac, entries=150, sequenceid=397, filesize=12.0 K 2024-12-02T21:15:32,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 8a7bf3289fb638014baa7fb12fa0d82c in 550ms, sequenceid=397, compaction requested=false 2024-12-02T21:15:32,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:32,738 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/65ad647c97d641fdbf213ff63e9737f4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/65ad647c97d641fdbf213ff63e9737f4 2024-12-02T21:15:32,749 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into 65ad647c97d641fdbf213ff63e9737f4(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:32,749 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:32,750 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=13, startTime=1733174132071; duration=0sec 2024-12-02T21:15:32,750 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:32,750 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:32,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:15:32,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:32,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:32,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:32,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:32,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:32,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:32,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/fbef109d91244169b388364d8dcf9189 is 50, key is test_row_0/A:col10/1733174132155/Put/seqid=0 2024-12-02T21:15:32,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741917_1093 (size=12301) 2024-12-02T21:15:32,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/fbef109d91244169b388364d8dcf9189 2024-12-02T21:15:32,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T21:15:32,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:32,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:32,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174132826, max timestamp=1733174132858Num uniques 859; 2024-12-02T21:15:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174132826, max timestamp=1733174132858Num uniques 859; 2024-12-02T21:15:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174132826, max timestamp=1733174132858Num uniques 859; 2024-12-02T21:15:32,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:32,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b4cf593ed42041559d9f3476e442b02d is 50, key is test_row_0/B:col10/1733174132155/Put/seqid=0 2024-12-02T21:15:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174192889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174192890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174192892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174192899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174192900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:32,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:32,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:32,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741918_1094 (size=12301) 2024-12-02T21:15:32,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b4cf593ed42041559d9f3476e442b02d 2024-12-02T21:15:32,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1ac7532e2094498783be51553ab475ad is 50, key is test_row_0/C:col10/1733174132155/Put/seqid=0 2024-12-02T21:15:33,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T21:15:33,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:33,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:33,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:33,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:33,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:33,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741919_1095 (size=12301) 2024-12-02T21:15:33,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1ac7532e2094498783be51553ab475ad 2024-12-02T21:15:33,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174193009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174193009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174193009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174193009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174193019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/fbef109d91244169b388364d8dcf9189 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbef109d91244169b388364d8dcf9189 2024-12-02T21:15:33,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbef109d91244169b388364d8dcf9189, entries=150, sequenceid=412, filesize=12.0 K 2024-12-02T21:15:33,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b4cf593ed42041559d9f3476e442b02d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b4cf593ed42041559d9f3476e442b02d 2024-12-02T21:15:33,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b4cf593ed42041559d9f3476e442b02d, entries=150, sequenceid=412, filesize=12.0 K 2024-12-02T21:15:33,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/1ac7532e2094498783be51553ab475ad as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1ac7532e2094498783be51553ab475ad 2024-12-02T21:15:33,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1ac7532e2094498783be51553ab475ad, entries=150, sequenceid=412, filesize=12.0 K 2024-12-02T21:15:33,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8a7bf3289fb638014baa7fb12fa0d82c in 278ms, sequenceid=412, compaction requested=true 2024-12-02T21:15:33,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:33,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:33,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:33,070 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:33,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:33,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:33,070 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:33,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:33,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:33,071 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:33,071 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:33,072 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:33,072 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b479258b65d24026bfde2f27e9ee565e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/52d032c9682c449d9a133e6aa5a28bc9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b4cf593ed42041559d9f3476e442b02d] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=36.9 K 2024-12-02T21:15:33,073 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:33,073 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:33,073 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:33,073 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/65ad647c97d641fdbf213ff63e9737f4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/9b3bbe820e9e4cafbdcc5863db36aee5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbef109d91244169b388364d8dcf9189] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=39.3 K 2024-12-02T21:15:33,074 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b479258b65d24026bfde2f27e9ee565e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733174131938 2024-12-02T21:15:33,075 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ad647c97d641fdbf213ff63e9737f4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733174131938 2024-12-02T21:15:33,076 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d032c9682c449d9a133e6aa5a28bc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733174132021 2024-12-02T21:15:33,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b3bbe820e9e4cafbdcc5863db36aee5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733174132021 2024-12-02T21:15:33,078 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbef109d91244169b388364d8dcf9189, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733174132152 2024-12-02T21:15:33,078 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b4cf593ed42041559d9f3476e442b02d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733174132152 2024-12-02T21:15:33,101 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:33,102 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/0a8a418e9c6b4c00bd6b4ef3672dc483 is 50, key is test_row_0/B:col10/1733174132155/Put/seqid=0 2024-12-02T21:15:33,110 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:33,110 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/d91a89cd84244bbe90346213a9a2c164 is 50, key is test_row_0/A:col10/1733174132155/Put/seqid=0 2024-12-02T21:15:33,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741920_1096 (size=13255) 2024-12-02T21:15:33,150 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/d91a89cd84244bbe90346213a9a2c164 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/d91a89cd84244bbe90346213a9a2c164 2024-12-02T21:15:33,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T21:15:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:33,156 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T21:15:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:33,161 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into d91a89cd84244bbe90346213a9a2c164(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:33,161 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:33,162 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=13, startTime=1733174133070; duration=0sec 2024-12-02T21:15:33,162 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:33,162 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:33,162 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:33,165 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:33,166 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:33,166 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:33,166 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/41c4e0506b324853bd1da1095bbfe7da, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d92e74d3f90c4f1482461fe4e05c90ac, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1ac7532e2094498783be51553ab475ad] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=36.8 K 2024-12-02T21:15:33,169 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c4e0506b324853bd1da1095bbfe7da, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733174131938 2024-12-02T21:15:33,170 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting d92e74d3f90c4f1482461fe4e05c90ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733174132021 2024-12-02T21:15:33,171 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ac7532e2094498783be51553ab475ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733174132152 2024-12-02T21:15:33,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741921_1097 (size=13255) 2024-12-02T21:15:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/50fbb5fe5a0445fe8552bb06bd8d8de7 is 50, key is test_row_0/A:col10/1733174132897/Put/seqid=0 2024-12-02T21:15:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T21:15:33,202 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:33,203 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/44baddbc5f9d4fc8af3c311a617678bc is 50, key is test_row_0/C:col10/1733174132155/Put/seqid=0 2024-12-02T21:15:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:33,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:33,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741922_1098 (size=12301) 2024-12-02T21:15:33,233 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/50fbb5fe5a0445fe8552bb06bd8d8de7 2024-12-02T21:15:33,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741923_1099 (size=13221) 2024-12-02T21:15:33,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174193232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174193234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174193235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174193237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174193249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/61cb404a204c46b39ac2f812fc2dec3a is 50, key is test_row_0/B:col10/1733174132897/Put/seqid=0 2024-12-02T21:15:33,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741924_1100 (size=12301) 2024-12-02T21:15:33,270 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/61cb404a204c46b39ac2f812fc2dec3a 2024-12-02T21:15:33,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/7dca620a69804aa39e9b7e132748bcc9 is 50, key is test_row_0/C:col10/1733174132897/Put/seqid=0 2024-12-02T21:15:33,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741925_1101 (size=12301) 2024-12-02T21:15:33,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174193349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174193349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174193358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174193358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174193359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174193558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174193559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174193569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174193569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174193569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,594 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/0a8a418e9c6b4c00bd6b4ef3672dc483 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/0a8a418e9c6b4c00bd6b4ef3672dc483 2024-12-02T21:15:33,603 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into 0a8a418e9c6b4c00bd6b4ef3672dc483(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:33,603 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:33,603 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=13, startTime=1733174133070; duration=0sec 2024-12-02T21:15:33,603 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:33,603 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:33,641 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/44baddbc5f9d4fc8af3c311a617678bc as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/44baddbc5f9d4fc8af3c311a617678bc 2024-12-02T21:15:33,649 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into 44baddbc5f9d4fc8af3c311a617678bc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:33,649 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:33,649 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=13, startTime=1733174133070; duration=0sec 2024-12-02T21:15:33,649 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:33,649 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:33,691 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/7dca620a69804aa39e9b7e132748bcc9 2024-12-02T21:15:33,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/50fbb5fe5a0445fe8552bb06bd8d8de7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/50fbb5fe5a0445fe8552bb06bd8d8de7 2024-12-02T21:15:33,706 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/50fbb5fe5a0445fe8552bb06bd8d8de7, entries=150, sequenceid=448, filesize=12.0 K 2024-12-02T21:15:33,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/61cb404a204c46b39ac2f812fc2dec3a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/61cb404a204c46b39ac2f812fc2dec3a 2024-12-02T21:15:33,715 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/61cb404a204c46b39ac2f812fc2dec3a, entries=150, sequenceid=448, filesize=12.0 K 2024-12-02T21:15:33,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/7dca620a69804aa39e9b7e132748bcc9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/7dca620a69804aa39e9b7e132748bcc9 2024-12-02T21:15:33,727 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/7dca620a69804aa39e9b7e132748bcc9, entries=150, sequenceid=448, filesize=12.0 K 2024-12-02T21:15:33,728 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 8a7bf3289fb638014baa7fb12fa0d82c in 572ms, sequenceid=448, compaction requested=false 2024-12-02T21:15:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-02T21:15:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-02T21:15:33,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-02T21:15:33,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6620 sec 2024-12-02T21:15:33,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6770 sec 2024-12-02T21:15:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:33,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:15:33,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:33,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:33,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:33,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:33,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:33,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:33,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/c558eaf6e9244212abe1dbb966267b02 is 50, key is test_row_0/A:col10/1733174133870/Put/seqid=0 2024-12-02T21:15:33,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741926_1102 (size=14741) 2024-12-02T21:15:33,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/c558eaf6e9244212abe1dbb966267b02 2024-12-02T21:15:33,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174193916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174133911, max timestamp=1733174133926Num uniques 859; 2024-12-02T21:15:33,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:33,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174133911, max timestamp=1733174133926Num uniques 859; 2024-12-02T21:15:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174193919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174133911, max timestamp=1733174133926Num uniques 859; 2024-12-02T21:15:33,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:33,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/92183a886e6747b38e9a23932a7d43e3 is 50, key is test_row_0/B:col10/1733174133870/Put/seqid=0 2024-12-02T21:15:33,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741927_1103 (size=12301) 2024-12-02T21:15:33,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/92183a886e6747b38e9a23932a7d43e3 2024-12-02T21:15:33,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/02548343f2f74a5c822c43cf04f0e82a is 50, key is test_row_0/C:col10/1733174133870/Put/seqid=0 2024-12-02T21:15:33,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174193981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174193986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:33,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174193989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:33,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:33,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741928_1104 (size=12301) 2024-12-02T21:15:34,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/02548343f2f74a5c822c43cf04f0e82a 2024-12-02T21:15:34,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/c558eaf6e9244212abe1dbb966267b02 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c558eaf6e9244212abe1dbb966267b02 2024-12-02T21:15:34,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c558eaf6e9244212abe1dbb966267b02, entries=200, sequenceid=462, filesize=14.4 K 2024-12-02T21:15:34,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/92183a886e6747b38e9a23932a7d43e3 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/92183a886e6747b38e9a23932a7d43e3 2024-12-02T21:15:34,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174194039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174194039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/92183a886e6747b38e9a23932a7d43e3, entries=150, sequenceid=462, filesize=12.0 K 2024-12-02T21:15:34,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/02548343f2f74a5c822c43cf04f0e82a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/02548343f2f74a5c822c43cf04f0e82a 2024-12-02T21:15:34,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/02548343f2f74a5c822c43cf04f0e82a, entries=150, sequenceid=462, filesize=12.0 K 2024-12-02T21:15:34,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 8a7bf3289fb638014baa7fb12fa0d82c in 197ms, sequenceid=462, compaction requested=true 2024-12-02T21:15:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:34,069 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:34,069 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:34,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:34,071 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:34,071 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:34,071 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,072 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/0a8a418e9c6b4c00bd6b4ef3672dc483, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/61cb404a204c46b39ac2f812fc2dec3a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/92183a886e6747b38e9a23932a7d43e3] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=37.0 K 2024-12-02T21:15:34,072 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:34,072 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:34,072 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,072 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/d91a89cd84244bbe90346213a9a2c164, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/50fbb5fe5a0445fe8552bb06bd8d8de7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c558eaf6e9244212abe1dbb966267b02] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=39.4 K 2024-12-02T21:15:34,072 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a8a418e9c6b4c00bd6b4ef3672dc483, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733174132152 2024-12-02T21:15:34,073 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting d91a89cd84244bbe90346213a9a2c164, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733174132152 2024-12-02T21:15:34,073 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50fbb5fe5a0445fe8552bb06bd8d8de7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733174132889 2024-12-02T21:15:34,073 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 61cb404a204c46b39ac2f812fc2dec3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733174132889 2024-12-02T21:15:34,074 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting c558eaf6e9244212abe1dbb966267b02, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733174133229 2024-12-02T21:15:34,074 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 92183a886e6747b38e9a23932a7d43e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733174133229 2024-12-02T21:15:34,088 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:34,089 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/bb5cf6c43289432dab4e0fe5d321ca8e is 50, key is test_row_0/A:col10/1733174133870/Put/seqid=0 2024-12-02T21:15:34,094 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:34,095 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/e6a6242c6614463ba9fc5fa81b018348 is 50, key is test_row_0/B:col10/1733174133870/Put/seqid=0 2024-12-02T21:15:34,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T21:15:34,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:34,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:34,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:34,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:34,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:34,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:34,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/eb2b7a4d58bc4981b889473d4ef8afd5 is 50, key is test_row_0/A:col10/1733174133975/Put/seqid=0 2024-12-02T21:15:34,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741929_1105 (size=13357) 2024-12-02T21:15:34,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174194117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174194118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,129 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/e6a6242c6614463ba9fc5fa81b018348 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e6a6242c6614463ba9fc5fa81b018348 2024-12-02T21:15:34,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174194126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,141 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into e6a6242c6614463ba9fc5fa81b018348(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:34,141 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:34,141 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=13, startTime=1733174134069; duration=0sec 2024-12-02T21:15:34,141 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:34,141 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:34,141 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:34,144 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:34,145 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:34,145 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,145 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/44baddbc5f9d4fc8af3c311a617678bc, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/7dca620a69804aa39e9b7e132748bcc9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/02548343f2f74a5c822c43cf04f0e82a] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=36.9 K 2024-12-02T21:15:34,146 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 44baddbc5f9d4fc8af3c311a617678bc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733174132152 2024-12-02T21:15:34,146 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dca620a69804aa39e9b7e132748bcc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733174132889 2024-12-02T21:15:34,147 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 02548343f2f74a5c822c43cf04f0e82a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733174133229 2024-12-02T21:15:34,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741930_1106 (size=13357) 2024-12-02T21:15:34,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741931_1107 (size=14741) 2024-12-02T21:15:34,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/eb2b7a4d58bc4981b889473d4ef8afd5 2024-12-02T21:15:34,186 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:34,187 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/86d278e3134f47819b643fefd9576595 is 50, key is test_row_0/C:col10/1733174133870/Put/seqid=0 2024-12-02T21:15:34,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/67fefaed5504420aa589fedfe1485d1f is 50, key is test_row_0/B:col10/1733174133975/Put/seqid=0 2024-12-02T21:15:34,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741932_1108 (size=13323) 2024-12-02T21:15:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T21:15:34,209 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:34,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:34,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-02T21:15:34,213 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T21:15:34,214 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/86d278e3134f47819b643fefd9576595 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/86d278e3134f47819b643fefd9576595 2024-12-02T21:15:34,214 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:34,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:34,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741933_1109 (size=12301) 2024-12-02T21:15:34,225 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into 86d278e3134f47819b643fefd9576595(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:34,225 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:34,225 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=13, startTime=1733174134069; duration=0sec 2024-12-02T21:15:34,226 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:34,226 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:34,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174194229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174194230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174194239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174194249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174194259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T21:15:34,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:34,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:34,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174194439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174194439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174194449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T21:15:34,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174194559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174194571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,575 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/bb5cf6c43289432dab4e0fe5d321ca8e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb5cf6c43289432dab4e0fe5d321ca8e 2024-12-02T21:15:34,586 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into bb5cf6c43289432dab4e0fe5d321ca8e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:34,586 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:34,586 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=13, startTime=1733174134069; duration=0sec 2024-12-02T21:15:34,586 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:34,586 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:34,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/67fefaed5504420aa589fedfe1485d1f 2024-12-02T21:15:34,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/c3b848742ef343c581050effefd1f353 is 50, key is test_row_0/C:col10/1733174133975/Put/seqid=0 2024-12-02T21:15:34,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741934_1110 (size=12301) 2024-12-02T21:15:34,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/c3b848742ef343c581050effefd1f353 2024-12-02T21:15:34,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/eb2b7a4d58bc4981b889473d4ef8afd5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/eb2b7a4d58bc4981b889473d4ef8afd5 2024-12-02T21:15:34,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:34,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/eb2b7a4d58bc4981b889473d4ef8afd5, entries=200, sequenceid=499, filesize=14.4 K 2024-12-02T21:15:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/67fefaed5504420aa589fedfe1485d1f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/67fefaed5504420aa589fedfe1485d1f 2024-12-02T21:15:34,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/67fefaed5504420aa589fedfe1485d1f, entries=150, sequenceid=499, filesize=12.0 K 2024-12-02T21:15:34,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/c3b848742ef343c581050effefd1f353 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c3b848742ef343c581050effefd1f353 2024-12-02T21:15:34,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c3b848742ef343c581050effefd1f353, entries=150, sequenceid=499, filesize=12.0 K 2024-12-02T21:15:34,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 8a7bf3289fb638014baa7fb12fa0d82c in 602ms, sequenceid=499, compaction requested=false 2024-12-02T21:15:34,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:34,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:15:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:34,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:34,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/c6402c0d689c43b783a23fffe203573d is 50, key is test_row_0/A:col10/1733174134123/Put/seqid=0 2024-12-02T21:15:34,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741935_1111 (size=14741) 2024-12-02T21:15:34,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/c6402c0d689c43b783a23fffe203573d 2024-12-02T21:15:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174194791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174134776, max timestamp=1733174134797Num uniques 859; 2024-12-02T21:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174134776, max timestamp=1733174134797Num uniques 859; 2024-12-02T21:15:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174134776, max timestamp=1733174134797Num uniques 859; 2024-12-02T21:15:34,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:34,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/6051dcc454ee49349027c5e153ce9db6 is 50, key is test_row_0/B:col10/1733174134123/Put/seqid=0 2024-12-02T21:15:34,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T21:15:34,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174194847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174194847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741936_1112 (size=12301) 2024-12-02T21:15:34,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/6051dcc454ee49349027c5e153ce9db6 2024-12-02T21:15:34,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/47d2d5a4f4ab4c9db75d23585dfa3ac7 is 50, key is test_row_0/C:col10/1733174134123/Put/seqid=0 2024-12-02T21:15:34,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174194909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741937_1113 (size=12301) 2024-12-02T21:15:34,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/47d2d5a4f4ab4c9db75d23585dfa3ac7 2024-12-02T21:15:34,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/c6402c0d689c43b783a23fffe203573d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c6402c0d689c43b783a23fffe203573d 2024-12-02T21:15:34,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c6402c0d689c43b783a23fffe203573d, entries=200, sequenceid=513, filesize=14.4 K 2024-12-02T21:15:34,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174194959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174194959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:34,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/6051dcc454ee49349027c5e153ce9db6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/6051dcc454ee49349027c5e153ce9db6 2024-12-02T21:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:34,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/6051dcc454ee49349027c5e153ce9db6, entries=150, sequenceid=513, filesize=12.0 K 2024-12-02T21:15:34,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/47d2d5a4f4ab4c9db75d23585dfa3ac7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/47d2d5a4f4ab4c9db75d23585dfa3ac7 2024-12-02T21:15:34,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:34,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:34,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/47d2d5a4f4ab4c9db75d23585dfa3ac7, entries=150, sequenceid=513, filesize=12.0 K 2024-12-02T21:15:34,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8a7bf3289fb638014baa7fb12fa0d82c in 235ms, sequenceid=513, compaction requested=true 2024-12-02T21:15:34,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:34,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:34,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:34,989 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:34,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:34,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:34,989 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:34,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:34,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:34,991 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:34,991 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:34,991 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,991 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb5cf6c43289432dab4e0fe5d321ca8e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/eb2b7a4d58bc4981b889473d4ef8afd5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c6402c0d689c43b783a23fffe203573d] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=41.8 K 2024-12-02T21:15:34,991 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:34,992 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:34,992 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:34,992 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e6a6242c6614463ba9fc5fa81b018348, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/67fefaed5504420aa589fedfe1485d1f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/6051dcc454ee49349027c5e153ce9db6] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=37.1 K 2024-12-02T21:15:34,992 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5cf6c43289432dab4e0fe5d321ca8e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733174133229 2024-12-02T21:15:34,992 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a6242c6614463ba9fc5fa81b018348, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733174133229 2024-12-02T21:15:34,993 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 67fefaed5504420aa589fedfe1485d1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1733174133975 2024-12-02T21:15:34,993 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb2b7a4d58bc4981b889473d4ef8afd5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1733174133975 2024-12-02T21:15:34,993 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6051dcc454ee49349027c5e153ce9db6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733174134123 2024-12-02T21:15:34,993 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6402c0d689c43b783a23fffe203573d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733174134116 2024-12-02T21:15:35,021 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:35,021 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/9b4ae21396ea4d178cc88a02502b1569 is 50, key is test_row_0/B:col10/1733174134123/Put/seqid=0 2024-12-02T21:15:35,026 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:35,027 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/2973bcd51c194cb8a0552cbbdc0d992f is 50, key is test_row_0/A:col10/1733174134123/Put/seqid=0 2024-12-02T21:15:35,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741938_1114 (size=13459) 2024-12-02T21:15:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:35,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T21:15:35,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:35,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:35,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:35,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:35,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:35,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:35,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741939_1115 (size=13459) 2024-12-02T21:15:35,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/8f915e523ff2423681fb332a761c5209 is 50, key is test_row_0/A:col10/1733174134845/Put/seqid=0 2024-12-02T21:15:35,095 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/2973bcd51c194cb8a0552cbbdc0d992f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/2973bcd51c194cb8a0552cbbdc0d992f 2024-12-02T21:15:35,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174195100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,105 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into 2973bcd51c194cb8a0552cbbdc0d992f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:35,106 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:35,106 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=13, startTime=1733174134989; duration=0sec 2024-12-02T21:15:35,106 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:35,106 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:35,106 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:35,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174195102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,108 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:35,108 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:35,109 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,109 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/86d278e3134f47819b643fefd9576595, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c3b848742ef343c581050effefd1f353, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/47d2d5a4f4ab4c9db75d23585dfa3ac7] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=37.0 K 2024-12-02T21:15:35,109 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86d278e3134f47819b643fefd9576595, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733174133229 2024-12-02T21:15:35,110 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3b848742ef343c581050effefd1f353, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1733174133975 2024-12-02T21:15:35,110 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d2d5a4f4ab4c9db75d23585dfa3ac7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733174134123 2024-12-02T21:15:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174195118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741940_1116 (size=14741) 2024-12-02T21:15:35,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,149 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:35,150 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/f9f2c7356a0d48e49ff6bdd592745c1e is 50, key is test_row_0/C:col10/1733174134123/Put/seqid=0 2024-12-02T21:15:35,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741941_1117 (size=13425) 2024-12-02T21:15:35,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174195171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174195171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174195209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174195219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T21:15:35,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174195428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174195429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174195429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,467 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/9b4ae21396ea4d178cc88a02502b1569 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/9b4ae21396ea4d178cc88a02502b1569 2024-12-02T21:15:35,472 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into 9b4ae21396ea4d178cc88a02502b1569(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:35,472 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:35,473 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=13, startTime=1733174134989; duration=0sec 2024-12-02T21:15:35,473 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:35,473 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:35,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174195478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174195478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/8f915e523ff2423681fb332a761c5209 2024-12-02T21:15:35,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b7bf8c8240ef45079345c80122e096b9 is 50, key is test_row_0/B:col10/1733174134845/Put/seqid=0 2024-12-02T21:15:35,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741942_1118 (size=12301) 2024-12-02T21:15:35,578 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/f9f2c7356a0d48e49ff6bdd592745c1e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/f9f2c7356a0d48e49ff6bdd592745c1e 2024-12-02T21:15:35,584 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into f9f2c7356a0d48e49ff6bdd592745c1e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:35,584 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:35,584 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=13, startTime=1733174134989; duration=0sec 2024-12-02T21:15:35,584 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:35,584 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:35,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:35,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:35,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174195739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174195739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:35,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:35,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:35,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:35,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:35,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:35,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174195938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b7bf8c8240ef45079345c80122e096b9 2024-12-02T21:15:35,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/318814ad6aee42c3b2817d0b6492dd05 is 50, key is test_row_0/C:col10/1733174134845/Put/seqid=0 2024-12-02T21:15:35,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174195989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174195989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:35,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:36,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741943_1119 (size=12301) 2024-12-02T21:15:36,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/318814ad6aee42c3b2817d0b6492dd05 2024-12-02T21:15:36,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/8f915e523ff2423681fb332a761c5209 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8f915e523ff2423681fb332a761c5209 2024-12-02T21:15:36,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8f915e523ff2423681fb332a761c5209, entries=200, sequenceid=549, filesize=14.4 K 2024-12-02T21:15:36,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/b7bf8c8240ef45079345c80122e096b9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b7bf8c8240ef45079345c80122e096b9 2024-12-02T21:15:36,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b7bf8c8240ef45079345c80122e096b9, entries=150, sequenceid=549, filesize=12.0 K 2024-12-02T21:15:36,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/318814ad6aee42c3b2817d0b6492dd05 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/318814ad6aee42c3b2817d0b6492dd05 2024-12-02T21:15:36,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/318814ad6aee42c3b2817d0b6492dd05, entries=150, sequenceid=549, filesize=12.0 K 2024-12-02T21:15:36,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:36,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:36,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 8a7bf3289fb638014baa7fb12fa0d82c in 994ms, sequenceid=549, compaction requested=false 2024-12-02T21:15:36,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:36,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T21:15:36,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:36,220 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:15:36,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:36,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:36,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:36,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:36,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:36,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:36,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/bb0311a04eb04d2abffc3f4087217c0e is 50, key is test_row_0/A:col10/1733174135100/Put/seqid=0 2024-12-02T21:15:36,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741944_1120 (size=12301) 2024-12-02T21:15:36,232 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/bb0311a04eb04d2abffc3f4087217c0e 2024-12-02T21:15:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/223ea75668f84a4a96bfce94e4006881 is 50, key is test_row_0/B:col10/1733174135100/Put/seqid=0 2024-12-02T21:15:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:36,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:36,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741945_1121 (size=12301) 2024-12-02T21:15:36,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174136287, max timestamp=1733174136301Num uniques 859; 2024-12-02T21:15:36,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174136287, max timestamp=1733174136301Num uniques 859; 2024-12-02T21:15:36,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174136287, max timestamp=1733174136301Num uniques 859; 2024-12-02T21:15:36,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:36,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:36,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T21:15:36,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174196361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:36,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174196362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:36,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174196479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:36,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174196479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:36,675 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/223ea75668f84a4a96bfce94e4006881 2024-12-02T21:15:36,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/aed5edfcf2dd4b7a842a641c644e8f8f is 50, key is test_row_0/C:col10/1733174135100/Put/seqid=0 2024-12-02T21:15:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174196689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:36,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741946_1122 (size=12301) 2024-12-02T21:15:36,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174196689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:36,692 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/aed5edfcf2dd4b7a842a641c644e8f8f 2024-12-02T21:15:36,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/bb0311a04eb04d2abffc3f4087217c0e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb0311a04eb04d2abffc3f4087217c0e 2024-12-02T21:15:36,704 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb0311a04eb04d2abffc3f4087217c0e, entries=150, sequenceid=562, filesize=12.0 K 2024-12-02T21:15:36,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/223ea75668f84a4a96bfce94e4006881 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/223ea75668f84a4a96bfce94e4006881 2024-12-02T21:15:36,714 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/223ea75668f84a4a96bfce94e4006881, entries=150, sequenceid=562, filesize=12.0 K 2024-12-02T21:15:36,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/aed5edfcf2dd4b7a842a641c644e8f8f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/aed5edfcf2dd4b7a842a641c644e8f8f 2024-12-02T21:15:36,725 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/aed5edfcf2dd4b7a842a641c644e8f8f, entries=150, sequenceid=562, filesize=12.0 K 2024-12-02T21:15:36,726 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 8a7bf3289fb638014baa7fb12fa0d82c in 506ms, sequenceid=562, compaction requested=true 2024-12-02T21:15:36,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:36,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:36,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-02T21:15:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-02T21:15:36,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-02T21:15:36,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5140 sec 2024-12-02T21:15:36,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.5210 sec 2024-12-02T21:15:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:36,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-02T21:15:36,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:36,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T21:15:36,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:36,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T21:15:36,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:36,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T21:15:36,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/ac099ecf0a6a4f6e80a795062bea6e02 is 50, key is test_row_0/A:col10/1733174136950/Put/seqid=0 2024-12-02T21:15:36,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741947_1123 (size=14741) 2024-12-02T21:15:36,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174196984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:36,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:36,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174196999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174196999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174196999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,006 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 8a7bf3289fb638014baa7fb12fa0d82c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:37,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174197008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:37,009 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 8a7bf3289fb638014baa7fb12fa0d82c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174197098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174197309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/ac099ecf0a6a4f6e80a795062bea6e02 2024-12-02T21:15:37,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/a04723fc6cee480aa6ad615eddbaaf4b is 50, key is test_row_0/B:col10/1733174136950/Put/seqid=0 2024-12-02T21:15:37,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741948_1124 (size=12301) 2024-12-02T21:15:37,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/a04723fc6cee480aa6ad615eddbaaf4b 2024-12-02T21:15:37,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/84f3ec3d00c849a289cfa08d3a4a647a is 50, key is test_row_0/C:col10/1733174136950/Put/seqid=0 2024-12-02T21:15:37,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741949_1125 (size=12301) 2024-12-02T21:15:37,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174197509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174197519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174197629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:37,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/84f3ec3d00c849a289cfa08d3a4a647a 2024-12-02T21:15:37,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/ac099ecf0a6a4f6e80a795062bea6e02 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/ac099ecf0a6a4f6e80a795062bea6e02 2024-12-02T21:15:37,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/ac099ecf0a6a4f6e80a795062bea6e02, entries=200, sequenceid=600, filesize=14.4 K 2024-12-02T21:15:37,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/a04723fc6cee480aa6ad615eddbaaf4b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/a04723fc6cee480aa6ad615eddbaaf4b 2024-12-02T21:15:37,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/a04723fc6cee480aa6ad615eddbaaf4b, entries=150, sequenceid=600, filesize=12.0 K 2024-12-02T21:15:37,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/84f3ec3d00c849a289cfa08d3a4a647a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/84f3ec3d00c849a289cfa08d3a4a647a 2024-12-02T21:15:37,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/84f3ec3d00c849a289cfa08d3a4a647a, entries=150, sequenceid=600, filesize=12.0 K 2024-12-02T21:15:37,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 8a7bf3289fb638014baa7fb12fa0d82c in 993ms, sequenceid=600, compaction requested=true 2024-12-02T21:15:37,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:37,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:37,944 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:37,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:37,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:37,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:37,944 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:37,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a7bf3289fb638014baa7fb12fa0d82c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:37,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:37,947 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:37,947 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/B is initiating minor compaction (all files) 2024-12-02T21:15:37,947 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/B in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:37,947 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/9b4ae21396ea4d178cc88a02502b1569, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b7bf8c8240ef45079345c80122e096b9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/223ea75668f84a4a96bfce94e4006881, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/a04723fc6cee480aa6ad615eddbaaf4b] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=49.2 K 2024-12-02T21:15:37,947 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:37,947 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/A is initiating minor compaction (all files) 2024-12-02T21:15:37,947 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/A in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:37,948 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/2973bcd51c194cb8a0552cbbdc0d992f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8f915e523ff2423681fb332a761c5209, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb0311a04eb04d2abffc3f4087217c0e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/ac099ecf0a6a4f6e80a795062bea6e02] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=53.9 K 2024-12-02T21:15:37,948 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b4ae21396ea4d178cc88a02502b1569, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733174134123 2024-12-02T21:15:37,948 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2973bcd51c194cb8a0552cbbdc0d992f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733174134123 2024-12-02T21:15:37,948 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b7bf8c8240ef45079345c80122e096b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733174134813 2024-12-02T21:15:37,948 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f915e523ff2423681fb332a761c5209, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733174134813 2024-12-02T21:15:37,949 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 223ea75668f84a4a96bfce94e4006881, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733174135083 2024-12-02T21:15:37,949 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb0311a04eb04d2abffc3f4087217c0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733174135083 2024-12-02T21:15:37,949 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting a04723fc6cee480aa6ad615eddbaaf4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1733174136353 2024-12-02T21:15:37,949 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac099ecf0a6a4f6e80a795062bea6e02, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1733174136353 2024-12-02T21:15:37,972 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#A#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:37,973 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/141e4311f23e40789db5ed06886fd126 is 50, key is test_row_0/A:col10/1733174136950/Put/seqid=0 2024-12-02T21:15:37,978 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:37,979 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/981563a0ca514776bd1f492f31bd656e is 50, key is test_row_0/B:col10/1733174136950/Put/seqid=0 2024-12-02T21:15:38,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741950_1126 (size=13595) 2024-12-02T21:15:38,016 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/141e4311f23e40789db5ed06886fd126 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/141e4311f23e40789db5ed06886fd126 2024-12-02T21:15:38,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741951_1127 (size=13595) 2024-12-02T21:15:38,030 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/A of 8a7bf3289fb638014baa7fb12fa0d82c into 141e4311f23e40789db5ed06886fd126(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:38,030 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:38,030 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/A, priority=12, startTime=1733174137944; duration=0sec 2024-12-02T21:15:38,030 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:38,030 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:A 2024-12-02T21:15:38,031 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:38,031 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/981563a0ca514776bd1f492f31bd656e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/981563a0ca514776bd1f492f31bd656e 2024-12-02T21:15:38,035 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:38,035 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 8a7bf3289fb638014baa7fb12fa0d82c/C is initiating minor compaction (all files) 2024-12-02T21:15:38,036 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a7bf3289fb638014baa7fb12fa0d82c/C in TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:38,036 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/f9f2c7356a0d48e49ff6bdd592745c1e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/318814ad6aee42c3b2817d0b6492dd05, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/aed5edfcf2dd4b7a842a641c644e8f8f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/84f3ec3d00c849a289cfa08d3a4a647a] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp, totalSize=49.1 K 2024-12-02T21:15:38,036 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9f2c7356a0d48e49ff6bdd592745c1e, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733174134123 2024-12-02T21:15:38,037 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318814ad6aee42c3b2817d0b6492dd05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733174134813 2024-12-02T21:15:38,038 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting aed5edfcf2dd4b7a842a641c644e8f8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733174135083 2024-12-02T21:15:38,039 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84f3ec3d00c849a289cfa08d3a4a647a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1733174136353 2024-12-02T21:15:38,041 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/B of 8a7bf3289fb638014baa7fb12fa0d82c into 981563a0ca514776bd1f492f31bd656e(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:38,041 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:38,041 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/B, priority=12, startTime=1733174137944; duration=0sec 2024-12-02T21:15:38,041 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:38,042 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:B 2024-12-02T21:15:38,052 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a7bf3289fb638014baa7fb12fa0d82c#C#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:38,052 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/a3737ca9bdcb48769a9c8f2dce0b9f2d is 50, key is test_row_0/C:col10/1733174136950/Put/seqid=0 2024-12-02T21:15:38,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741952_1128 (size=13561) 2024-12-02T21:15:38,070 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/a3737ca9bdcb48769a9c8f2dce0b9f2d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/a3737ca9bdcb48769a9c8f2dce0b9f2d 2024-12-02T21:15:38,076 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a7bf3289fb638014baa7fb12fa0d82c/C of 8a7bf3289fb638014baa7fb12fa0d82c into a3737ca9bdcb48769a9c8f2dce0b9f2d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:38,076 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:38,076 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., storeName=8a7bf3289fb638014baa7fb12fa0d82c/C, priority=12, startTime=1733174137944; duration=0sec 2024-12-02T21:15:38,076 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:38,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a7bf3289fb638014baa7fb12fa0d82c:C 2024-12-02T21:15:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:38,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:15:38,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:38,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:38,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:38,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:38,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:38,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:38,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/4afcb43141364ebaadd1a89638e87cc3 is 50, key is test_row_0/A:col10/1733174138139/Put/seqid=0 2024-12-02T21:15:38,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741953_1129 (size=14741) 2024-12-02T21:15:38,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174138192, max timestamp=1733174138212Num uniques 859; 2024-12-02T21:15:38,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:38,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174138192, max timestamp=1733174138212Num uniques 859; 2024-12-02T21:15:38,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174138192, max timestamp=1733174138212Num uniques 859; 2024-12-02T21:15:38,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:38,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:38,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174198294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T21:15:38,369 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:38,370 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-02T21:15:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T21:15:38,374 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:38,375 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:38,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:38,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174198398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T21:15:38,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57740 deadline: 1733174198518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:38,519 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 8a7bf3289fb638014baa7fb12fa0d82c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:38,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T21:15:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:38,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57826 deadline: 1733174198539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:38,540 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 8a7bf3289fb638014baa7fb12fa0d82c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:38,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/4afcb43141364ebaadd1a89638e87cc3 2024-12-02T21:15:38,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/44c8b670f49741bbbc159cd4b3fe7b1d is 50, key is test_row_0/B:col10/1733174138139/Put/seqid=0 2024-12-02T21:15:38,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741954_1130 (size=12301) 2024-12-02T21:15:38,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/44c8b670f49741bbbc159cd4b3fe7b1d 2024-12-02T21:15:38,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174198608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:38,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/08b93cc06a8545fcac892de533a5e187 is 50, key is test_row_0/C:col10/1733174138139/Put/seqid=0 2024-12-02T21:15:38,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741955_1131 (size=12301) 2024-12-02T21:15:38,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T21:15:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:38,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T21:15:38,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T21:15:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:38,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,873 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-02T21:15:38,873 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:15:38,875 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-02T21:15:38,875 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:15:38,875 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-02T21:15:38,875 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:15:38,879 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:38,879 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:38,879 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:38,879 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:38,879 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:38,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:38,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:38,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:38,880 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:38,882 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-02T21:15:38,883 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:15:38,883 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:38,883 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:38,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:38,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57786 deadline: 1733174198919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:38,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:38,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:38,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T21:15:38,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:38,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:38,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:38,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T21:15:39,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57752 deadline: 1733174199028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:39,029 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 8a7bf3289fb638014baa7fb12fa0d82c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:39,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/08b93cc06a8545fcac892de533a5e187 2024-12-02T21:15:39,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/4afcb43141364ebaadd1a89638e87cc3 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4afcb43141364ebaadd1a89638e87cc3 2024-12-02T21:15:39,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57766 deadline: 1733174199039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:39,040 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 8a7bf3289fb638014baa7fb12fa0d82c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:15:39,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4afcb43141364ebaadd1a89638e87cc3, entries=200, sequenceid=613, filesize=14.4 K 2024-12-02T21:15:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/44c8b670f49741bbbc159cd4b3fe7b1d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/44c8b670f49741bbbc159cd4b3fe7b1d 2024-12-02T21:15:39,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/44c8b670f49741bbbc159cd4b3fe7b1d, entries=150, sequenceid=613, filesize=12.0 K 2024-12-02T21:15:39,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/08b93cc06a8545fcac892de533a5e187 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08b93cc06a8545fcac892de533a5e187 2024-12-02T21:15:39,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08b93cc06a8545fcac892de533a5e187, entries=150, sequenceid=613, filesize=12.0 K 2024-12-02T21:15:39,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8a7bf3289fb638014baa7fb12fa0d82c in 916ms, sequenceid=613, compaction requested=false 2024-12-02T21:15:39,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:39,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T21:15:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:39,143 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T21:15:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:39,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/fbb94c20e9c446b5b5d8781b855ace62 is 50, key is test_row_0/A:col10/1733174138291/Put/seqid=0 2024-12-02T21:15:39,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741956_1132 (size=12301) 2024-12-02T21:15:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:39,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. as already flushing 2024-12-02T21:15:39,431 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-02T21:15:39,431 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:15:39,431 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:39,432 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:39,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T21:15:39,553 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/fbb94c20e9c446b5b5d8781b855ace62 2024-12-02T21:15:39,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ef8cc04df9ce41db9b27ef56963d1723 is 50, key is test_row_0/B:col10/1733174138291/Put/seqid=0 2024-12-02T21:15:39,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741957_1133 (size=12301) 2024-12-02T21:15:39,969 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ef8cc04df9ce41db9b27ef56963d1723 2024-12-02T21:15:39,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/3959456a07994ae3a87275f4e33638db is 50, key is test_row_0/C:col10/1733174138291/Put/seqid=0 2024-12-02T21:15:39,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741958_1134 (size=12301) 2024-12-02T21:15:40,393 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/3959456a07994ae3a87275f4e33638db 2024-12-02T21:15:40,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/fbb94c20e9c446b5b5d8781b855ace62 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbb94c20e9c446b5b5d8781b855ace62 2024-12-02T21:15:40,410 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbb94c20e9c446b5b5d8781b855ace62, entries=150, sequenceid=649, filesize=12.0 K 2024-12-02T21:15:40,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/ef8cc04df9ce41db9b27ef56963d1723 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef8cc04df9ce41db9b27ef56963d1723 2024-12-02T21:15:40,418 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef8cc04df9ce41db9b27ef56963d1723, entries=150, sequenceid=649, filesize=12.0 K 2024-12-02T21:15:40,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/3959456a07994ae3a87275f4e33638db as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3959456a07994ae3a87275f4e33638db 2024-12-02T21:15:40,424 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3959456a07994ae3a87275f4e33638db, entries=150, sequenceid=649, filesize=12.0 K 2024-12-02T21:15:40,425 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=6.71 KB/6870 for 8a7bf3289fb638014baa7fb12fa0d82c in 1282ms, sequenceid=649, compaction requested=true 2024-12-02T21:15:40,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 8a7bf3289fb638014baa7fb12fa0d82c: 2024-12-02T21:15:40,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:40,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-02T21:15:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-02T21:15:40,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-02T21:15:40,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0500 sec 2024-12-02T21:15:40,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 2.0580 sec 2024-12-02T21:15:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T21:15:40,520 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:40,540 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-02T21:15:40,540 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:15:40,540 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:40,540 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:40,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:40,550 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-02T21:15:40,551 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:15:40,551 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:40,551 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:40,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:42,178 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T21:15:43,041 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-02T21:15:43,041 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:15:43,041 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:43,041 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:43,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:43,091 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-02T21:15:43,092 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:15:43,092 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:43,092 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:43,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:43,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T21:15:43,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 133 2024-12-02T21:15:43,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-12-02T21:15:43,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-02T21:15:43,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-02T21:15:43,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-12-02T21:15:43,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T21:15:43,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3327 2024-12-02T21:15:43,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3401 2024-12-02T21:15:43,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T21:15:43,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1555 2024-12-02T21:15:43,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4664 rows 2024-12-02T21:15:43,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1557 2024-12-02T21:15:43,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4670 rows 2024-12-02T21:15:43,094 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:15:43,094 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:15:43,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:43,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:43,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:15:43,102 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T21:15:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T21:15:43,109 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174143108"}]},"ts":"1733174143108"} 2024-12-02T21:15:43,110 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T21:15:43,111 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T21:15:43,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T21:15:43,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c, UNASSIGN}] 2024-12-02T21:15:43,117 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c, UNASSIGN 2024-12-02T21:15:43,119 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=8a7bf3289fb638014baa7fb12fa0d82c, regionState=CLOSING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:43,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c, UNASSIGN because future has completed 2024-12-02T21:15:43,121 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:15:43,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:15:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T21:15:43,280 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:43,280 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:15:43,280 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 8a7bf3289fb638014baa7fb12fa0d82c, disabling compactions & flushes 2024-12-02T21:15:43,280 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:43,280 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:43,280 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. after waiting 0 ms 2024-12-02T21:15:43,281 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:43,281 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 8a7bf3289fb638014baa7fb12fa0d82c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T21:15:43,281 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=A 2024-12-02T21:15:43,281 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:43,281 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=B 2024-12-02T21:15:43,281 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:43,281 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a7bf3289fb638014baa7fb12fa0d82c, store=C 2024-12-02T21:15:43,281 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:43,285 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/6c553d7ca59c49518d3eefbc5a1e2aef is 50, key is test_row_0/A:col10/1733174139429/Put/seqid=0 2024-12-02T21:15:43,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741959_1135 (size=12301) 2024-12-02T21:15:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T21:15:43,690 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/6c553d7ca59c49518d3eefbc5a1e2aef 2024-12-02T21:15:43,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1f41d40730d04c078564e05a1d65c88b is 50, key is test_row_0/B:col10/1733174139429/Put/seqid=0 2024-12-02T21:15:43,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741960_1136 (size=12301) 2024-12-02T21:15:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T21:15:44,108 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1f41d40730d04c078564e05a1d65c88b 2024-12-02T21:15:44,123 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/30ed97e54052434cb82b5a4f7f66c5f2 is 50, key is test_row_0/C:col10/1733174139429/Put/seqid=0 2024-12-02T21:15:44,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741961_1137 (size=12301) 2024-12-02T21:15:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T21:15:44,531 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/30ed97e54052434cb82b5a4f7f66c5f2 2024-12-02T21:15:44,544 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/A/6c553d7ca59c49518d3eefbc5a1e2aef as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/6c553d7ca59c49518d3eefbc5a1e2aef 2024-12-02T21:15:44,550 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/6c553d7ca59c49518d3eefbc5a1e2aef, entries=150, sequenceid=657, filesize=12.0 K 2024-12-02T21:15:44,552 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/B/1f41d40730d04c078564e05a1d65c88b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1f41d40730d04c078564e05a1d65c88b 2024-12-02T21:15:44,556 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1f41d40730d04c078564e05a1d65c88b, entries=150, sequenceid=657, filesize=12.0 K 2024-12-02T21:15:44,557 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/.tmp/C/30ed97e54052434cb82b5a4f7f66c5f2 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/30ed97e54052434cb82b5a4f7f66c5f2 2024-12-02T21:15:44,561 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/30ed97e54052434cb82b5a4f7f66c5f2, entries=150, sequenceid=657, filesize=12.0 K 2024-12-02T21:15:44,562 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8a7bf3289fb638014baa7fb12fa0d82c in 1281ms, sequenceid=657, compaction requested=true 2024-12-02T21:15:44,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/94bf6af8867d4d6ca83d3012c8dbb31f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/7f8409385b0545b2858e3b2c0758c648, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/275b560ad33542f184c9fbc850676f5c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fcbcbc76e90d4a5fb1a068fd7909d64e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3b53270d8eaa40538ae841818e62f871, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/91be804d3b34476496face96c08ff2d9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad235acbd0245979f3d7e43db8da1ef, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/28f683fa46f643ab84a4001f3de6a1c6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3805926d62c54abd972b8a1f20834fba, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bd2c375b21e74e88b7f8165f49dcd106, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1b632f74e3784f40ac52d1a4c3820db0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8b693e8a80204269b7127fc6985e609e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/85bbd861cece456aacb0fdcd76c2faa9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/70c3f3e853904921a89e0d0d4405fac2, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1f457b1c30ee4e779a4604428c99050e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad7bb5ec4674b28938028d99698473b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8355e1081e2f4f6f8de1fc0dcc4376e0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/37bf2f39c2004ef3936274905c810c87, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/09b90d372da74587b8b478217bf2c585, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/f2ebba3c0a6f4f968960008328699200, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4e33ffff965f4b7484f16f338549c8b6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/620e9702f6214ad39506bd866980f27f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3df2699b21eb40c1b9baeab81ef02918, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/642d07b435cd4bfa9829c6f0c5b7d7fd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/763f3f426bd84e0999570c0f6b69d995, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/65ad647c97d641fdbf213ff63e9737f4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/9b3bbe820e9e4cafbdcc5863db36aee5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/d91a89cd84244bbe90346213a9a2c164, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbef109d91244169b388364d8dcf9189, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/50fbb5fe5a0445fe8552bb06bd8d8de7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c558eaf6e9244212abe1dbb966267b02, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb5cf6c43289432dab4e0fe5d321ca8e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/eb2b7a4d58bc4981b889473d4ef8afd5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c6402c0d689c43b783a23fffe203573d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/2973bcd51c194cb8a0552cbbdc0d992f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8f915e523ff2423681fb332a761c5209, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb0311a04eb04d2abffc3f4087217c0e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/ac099ecf0a6a4f6e80a795062bea6e02] to archive 2024-12-02T21:15:44,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:15:44,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/94bf6af8867d4d6ca83d3012c8dbb31f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/94bf6af8867d4d6ca83d3012c8dbb31f 2024-12-02T21:15:44,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/7f8409385b0545b2858e3b2c0758c648 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/7f8409385b0545b2858e3b2c0758c648 2024-12-02T21:15:44,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/275b560ad33542f184c9fbc850676f5c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/275b560ad33542f184c9fbc850676f5c 2024-12-02T21:15:44,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fcbcbc76e90d4a5fb1a068fd7909d64e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fcbcbc76e90d4a5fb1a068fd7909d64e 2024-12-02T21:15:44,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3b53270d8eaa40538ae841818e62f871 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3b53270d8eaa40538ae841818e62f871 2024-12-02T21:15:44,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/91be804d3b34476496face96c08ff2d9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/91be804d3b34476496face96c08ff2d9 2024-12-02T21:15:44,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad235acbd0245979f3d7e43db8da1ef to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad235acbd0245979f3d7e43db8da1ef 2024-12-02T21:15:44,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/28f683fa46f643ab84a4001f3de6a1c6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/28f683fa46f643ab84a4001f3de6a1c6 2024-12-02T21:15:44,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3805926d62c54abd972b8a1f20834fba to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3805926d62c54abd972b8a1f20834fba 2024-12-02T21:15:44,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bd2c375b21e74e88b7f8165f49dcd106 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bd2c375b21e74e88b7f8165f49dcd106 2024-12-02T21:15:44,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1b632f74e3784f40ac52d1a4c3820db0 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1b632f74e3784f40ac52d1a4c3820db0 2024-12-02T21:15:44,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8b693e8a80204269b7127fc6985e609e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8b693e8a80204269b7127fc6985e609e 2024-12-02T21:15:44,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/85bbd861cece456aacb0fdcd76c2faa9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/85bbd861cece456aacb0fdcd76c2faa9 2024-12-02T21:15:44,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/70c3f3e853904921a89e0d0d4405fac2 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/70c3f3e853904921a89e0d0d4405fac2 2024-12-02T21:15:44,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1f457b1c30ee4e779a4604428c99050e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/1f457b1c30ee4e779a4604428c99050e 2024-12-02T21:15:44,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad7bb5ec4674b28938028d99698473b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/dad7bb5ec4674b28938028d99698473b 2024-12-02T21:15:44,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8355e1081e2f4f6f8de1fc0dcc4376e0 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8355e1081e2f4f6f8de1fc0dcc4376e0 2024-12-02T21:15:44,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/37bf2f39c2004ef3936274905c810c87 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/37bf2f39c2004ef3936274905c810c87 2024-12-02T21:15:44,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/09b90d372da74587b8b478217bf2c585 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/09b90d372da74587b8b478217bf2c585 2024-12-02T21:15:44,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/f2ebba3c0a6f4f968960008328699200 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/f2ebba3c0a6f4f968960008328699200 2024-12-02T21:15:44,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4e33ffff965f4b7484f16f338549c8b6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4e33ffff965f4b7484f16f338549c8b6 2024-12-02T21:15:44,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/620e9702f6214ad39506bd866980f27f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/620e9702f6214ad39506bd866980f27f 2024-12-02T21:15:44,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3df2699b21eb40c1b9baeab81ef02918 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/3df2699b21eb40c1b9baeab81ef02918 2024-12-02T21:15:44,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/642d07b435cd4bfa9829c6f0c5b7d7fd to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/642d07b435cd4bfa9829c6f0c5b7d7fd 2024-12-02T21:15:44,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/763f3f426bd84e0999570c0f6b69d995 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/763f3f426bd84e0999570c0f6b69d995 2024-12-02T21:15:44,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/65ad647c97d641fdbf213ff63e9737f4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/65ad647c97d641fdbf213ff63e9737f4 2024-12-02T21:15:44,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/9b3bbe820e9e4cafbdcc5863db36aee5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/9b3bbe820e9e4cafbdcc5863db36aee5 2024-12-02T21:15:44,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/d91a89cd84244bbe90346213a9a2c164 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/d91a89cd84244bbe90346213a9a2c164 2024-12-02T21:15:44,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbef109d91244169b388364d8dcf9189 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbef109d91244169b388364d8dcf9189 2024-12-02T21:15:44,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/50fbb5fe5a0445fe8552bb06bd8d8de7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/50fbb5fe5a0445fe8552bb06bd8d8de7 2024-12-02T21:15:44,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c558eaf6e9244212abe1dbb966267b02 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c558eaf6e9244212abe1dbb966267b02 2024-12-02T21:15:44,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb5cf6c43289432dab4e0fe5d321ca8e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb5cf6c43289432dab4e0fe5d321ca8e 2024-12-02T21:15:44,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/eb2b7a4d58bc4981b889473d4ef8afd5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/eb2b7a4d58bc4981b889473d4ef8afd5 2024-12-02T21:15:44,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c6402c0d689c43b783a23fffe203573d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/c6402c0d689c43b783a23fffe203573d 2024-12-02T21:15:44,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/2973bcd51c194cb8a0552cbbdc0d992f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/2973bcd51c194cb8a0552cbbdc0d992f 2024-12-02T21:15:44,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8f915e523ff2423681fb332a761c5209 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/8f915e523ff2423681fb332a761c5209 2024-12-02T21:15:44,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb0311a04eb04d2abffc3f4087217c0e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/bb0311a04eb04d2abffc3f4087217c0e 2024-12-02T21:15:44,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/ac099ecf0a6a4f6e80a795062bea6e02 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/ac099ecf0a6a4f6e80a795062bea6e02 2024-12-02T21:15:44,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f5a9ccbc0bc043848a4b197a6acee5d8, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/cbd7c09e6af84cf1b5dc43fb77f77ffe, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e7220d0952e7483685da62dd17d206e1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1916bc7cbbcc4e498046b49e325824aa, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/d578fd2204d04a16a997ce5ff44f8780, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ecca1c68c2114ba2b891df27a179dfb0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1467ca13c0654dc7b2225a1fa01be164, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/c731b561644847bebbe6957cb5a0f01d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ec402734ad904a42bc02c298cc3f61f7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7010577b378b43deade32bae95ba9f50, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e560f79de93a4d9b8e8515036cb57e34, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5daff2de87ea486b8cb1ac93243a1aad, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5f42840e0e1f4447ac20d59321cde061, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/8468001cbed04e78a845a7bb0821fd65, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef47922a03e64a14b06477db76767250, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/341ce53532184d019ed4b73491e1ab19, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5780dfd920b84f4487ad7ff72b681e63, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/57f7317e8dc84c82b1f97495899ceda5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/209235fc0c114ae5957a900cd5c69d5e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b8f36f1048854e52984e55a92414fe87, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b67135bd136d4db1bd059e3987915910, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7c617cd82ea147d68c25d86ca56f4717, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1e0c64e7910a452db695de1660e25826, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/bf2d0814b00f495f86f830c5137fa911, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b479258b65d24026bfde2f27e9ee565e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f063c527cdea4cc0a756ba63e1d99523, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/52d032c9682c449d9a133e6aa5a28bc9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/0a8a418e9c6b4c00bd6b4ef3672dc483, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b4cf593ed42041559d9f3476e442b02d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/61cb404a204c46b39ac2f812fc2dec3a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e6a6242c6614463ba9fc5fa81b018348, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/92183a886e6747b38e9a23932a7d43e3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/67fefaed5504420aa589fedfe1485d1f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/9b4ae21396ea4d178cc88a02502b1569, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/6051dcc454ee49349027c5e153ce9db6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b7bf8c8240ef45079345c80122e096b9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/223ea75668f84a4a96bfce94e4006881, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/a04723fc6cee480aa6ad615eddbaaf4b] to archive 2024-12-02T21:15:44,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:15:44,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f5a9ccbc0bc043848a4b197a6acee5d8 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f5a9ccbc0bc043848a4b197a6acee5d8 2024-12-02T21:15:44,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/cbd7c09e6af84cf1b5dc43fb77f77ffe to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/cbd7c09e6af84cf1b5dc43fb77f77ffe 2024-12-02T21:15:44,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e7220d0952e7483685da62dd17d206e1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e7220d0952e7483685da62dd17d206e1 2024-12-02T21:15:44,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1916bc7cbbcc4e498046b49e325824aa to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1916bc7cbbcc4e498046b49e325824aa 2024-12-02T21:15:44,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/d578fd2204d04a16a997ce5ff44f8780 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/d578fd2204d04a16a997ce5ff44f8780 2024-12-02T21:15:44,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ecca1c68c2114ba2b891df27a179dfb0 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ecca1c68c2114ba2b891df27a179dfb0 2024-12-02T21:15:44,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1467ca13c0654dc7b2225a1fa01be164 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1467ca13c0654dc7b2225a1fa01be164 2024-12-02T21:15:44,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/c731b561644847bebbe6957cb5a0f01d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/c731b561644847bebbe6957cb5a0f01d 2024-12-02T21:15:44,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ec402734ad904a42bc02c298cc3f61f7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ec402734ad904a42bc02c298cc3f61f7 2024-12-02T21:15:44,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7010577b378b43deade32bae95ba9f50 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7010577b378b43deade32bae95ba9f50 2024-12-02T21:15:44,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e560f79de93a4d9b8e8515036cb57e34 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e560f79de93a4d9b8e8515036cb57e34 2024-12-02T21:15:44,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5daff2de87ea486b8cb1ac93243a1aad to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5daff2de87ea486b8cb1ac93243a1aad 2024-12-02T21:15:44,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5f42840e0e1f4447ac20d59321cde061 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5f42840e0e1f4447ac20d59321cde061 2024-12-02T21:15:44,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/8468001cbed04e78a845a7bb0821fd65 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/8468001cbed04e78a845a7bb0821fd65 2024-12-02T21:15:44,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef47922a03e64a14b06477db76767250 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef47922a03e64a14b06477db76767250 2024-12-02T21:15:44,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/341ce53532184d019ed4b73491e1ab19 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/341ce53532184d019ed4b73491e1ab19 2024-12-02T21:15:44,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5780dfd920b84f4487ad7ff72b681e63 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/5780dfd920b84f4487ad7ff72b681e63 2024-12-02T21:15:44,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/57f7317e8dc84c82b1f97495899ceda5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/57f7317e8dc84c82b1f97495899ceda5 2024-12-02T21:15:44,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/209235fc0c114ae5957a900cd5c69d5e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/209235fc0c114ae5957a900cd5c69d5e 2024-12-02T21:15:44,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b8f36f1048854e52984e55a92414fe87 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b8f36f1048854e52984e55a92414fe87 2024-12-02T21:15:44,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b67135bd136d4db1bd059e3987915910 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b67135bd136d4db1bd059e3987915910 2024-12-02T21:15:44,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7c617cd82ea147d68c25d86ca56f4717 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/7c617cd82ea147d68c25d86ca56f4717 2024-12-02T21:15:44,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1e0c64e7910a452db695de1660e25826 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1e0c64e7910a452db695de1660e25826 2024-12-02T21:15:44,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/bf2d0814b00f495f86f830c5137fa911 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/bf2d0814b00f495f86f830c5137fa911 2024-12-02T21:15:44,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b479258b65d24026bfde2f27e9ee565e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b479258b65d24026bfde2f27e9ee565e 2024-12-02T21:15:44,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f063c527cdea4cc0a756ba63e1d99523 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/f063c527cdea4cc0a756ba63e1d99523 2024-12-02T21:15:44,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/52d032c9682c449d9a133e6aa5a28bc9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/52d032c9682c449d9a133e6aa5a28bc9 2024-12-02T21:15:44,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/0a8a418e9c6b4c00bd6b4ef3672dc483 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/0a8a418e9c6b4c00bd6b4ef3672dc483 2024-12-02T21:15:44,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b4cf593ed42041559d9f3476e442b02d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b4cf593ed42041559d9f3476e442b02d 2024-12-02T21:15:44,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/61cb404a204c46b39ac2f812fc2dec3a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/61cb404a204c46b39ac2f812fc2dec3a 2024-12-02T21:15:44,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e6a6242c6614463ba9fc5fa81b018348 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/e6a6242c6614463ba9fc5fa81b018348 2024-12-02T21:15:44,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/92183a886e6747b38e9a23932a7d43e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/92183a886e6747b38e9a23932a7d43e3 2024-12-02T21:15:44,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/67fefaed5504420aa589fedfe1485d1f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/67fefaed5504420aa589fedfe1485d1f 2024-12-02T21:15:44,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/9b4ae21396ea4d178cc88a02502b1569 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/9b4ae21396ea4d178cc88a02502b1569 2024-12-02T21:15:44,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/6051dcc454ee49349027c5e153ce9db6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/6051dcc454ee49349027c5e153ce9db6 2024-12-02T21:15:44,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b7bf8c8240ef45079345c80122e096b9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/b7bf8c8240ef45079345c80122e096b9 2024-12-02T21:15:44,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/223ea75668f84a4a96bfce94e4006881 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/223ea75668f84a4a96bfce94e4006881 2024-12-02T21:15:44,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/a04723fc6cee480aa6ad615eddbaaf4b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/a04723fc6cee480aa6ad615eddbaaf4b 2024-12-02T21:15:44,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d6bf36addb8846039e05bb47c8c30aa4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/209aa294117e4e87a756dd18d2a5470b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d84ab05e9b1149639de6f250d134ffee, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/943e562c6d204f608fe53a603631f495, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3475bfa9db064ec48375e48f61496bfb, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e260c8c6fdc94272826f438100a55ec7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b3b8a3776caf404888c44c3efa02b266, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/872d788bb3a44a84a6c4671663f4558b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1614254582cc49219a8d064bf0546267, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/5ccc547ca4e94a60b5eac0e55eb2f998, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/406cf57995d045939a40b4a9cf7ebf44, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c80384a7352f45c58bcaca88fd26c78a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/526225d4284a462e83b1fe2755614952, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/103f3165486545c198be94b07719e8b0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1f93b847bb4c45ff8dc54f7f7c7405b6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1d5cd51bfc634ac885c8a59beb1a9e79, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08425a8f92d0431caf714f1013641b78, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/9e4f30a1093a4514be1a7ccd33b71a46, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/37906e1ab4ff4bba912aa868bdd8969c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/23d6a6a62a2743b78d12fd734aa57d7f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e3f02059ccc945858094d5729e614bfd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/ea90d3ac15e24839a22c6c98bf51c564, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b8871e85a30c49ad95f1d51a516eff1e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/41c4e0506b324853bd1da1095bbfe7da, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e75de9370d6049aea4bfe4a4c289688a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d92e74d3f90c4f1482461fe4e05c90ac, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/44baddbc5f9d4fc8af3c311a617678bc, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1ac7532e2094498783be51553ab475ad, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/7dca620a69804aa39e9b7e132748bcc9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/86d278e3134f47819b643fefd9576595, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/02548343f2f74a5c822c43cf04f0e82a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c3b848742ef343c581050effefd1f353, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/f9f2c7356a0d48e49ff6bdd592745c1e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/47d2d5a4f4ab4c9db75d23585dfa3ac7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/318814ad6aee42c3b2817d0b6492dd05, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/aed5edfcf2dd4b7a842a641c644e8f8f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/84f3ec3d00c849a289cfa08d3a4a647a] to archive 2024-12-02T21:15:44,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:15:44,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d6bf36addb8846039e05bb47c8c30aa4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d6bf36addb8846039e05bb47c8c30aa4 2024-12-02T21:15:44,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/209aa294117e4e87a756dd18d2a5470b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/209aa294117e4e87a756dd18d2a5470b 2024-12-02T21:15:44,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d84ab05e9b1149639de6f250d134ffee to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d84ab05e9b1149639de6f250d134ffee 2024-12-02T21:15:44,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/943e562c6d204f608fe53a603631f495 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/943e562c6d204f608fe53a603631f495 2024-12-02T21:15:44,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3475bfa9db064ec48375e48f61496bfb to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3475bfa9db064ec48375e48f61496bfb 2024-12-02T21:15:44,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e260c8c6fdc94272826f438100a55ec7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e260c8c6fdc94272826f438100a55ec7 2024-12-02T21:15:44,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b3b8a3776caf404888c44c3efa02b266 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b3b8a3776caf404888c44c3efa02b266 2024-12-02T21:15:44,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/872d788bb3a44a84a6c4671663f4558b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/872d788bb3a44a84a6c4671663f4558b 2024-12-02T21:15:44,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1614254582cc49219a8d064bf0546267 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1614254582cc49219a8d064bf0546267 2024-12-02T21:15:44,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/5ccc547ca4e94a60b5eac0e55eb2f998 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/5ccc547ca4e94a60b5eac0e55eb2f998 2024-12-02T21:15:44,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/406cf57995d045939a40b4a9cf7ebf44 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/406cf57995d045939a40b4a9cf7ebf44 2024-12-02T21:15:44,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c80384a7352f45c58bcaca88fd26c78a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c80384a7352f45c58bcaca88fd26c78a 2024-12-02T21:15:44,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/526225d4284a462e83b1fe2755614952 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/526225d4284a462e83b1fe2755614952 2024-12-02T21:15:44,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/103f3165486545c198be94b07719e8b0 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/103f3165486545c198be94b07719e8b0 2024-12-02T21:15:44,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1f93b847bb4c45ff8dc54f7f7c7405b6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1f93b847bb4c45ff8dc54f7f7c7405b6 2024-12-02T21:15:44,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1d5cd51bfc634ac885c8a59beb1a9e79 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1d5cd51bfc634ac885c8a59beb1a9e79 2024-12-02T21:15:44,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08425a8f92d0431caf714f1013641b78 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08425a8f92d0431caf714f1013641b78 2024-12-02T21:15:44,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/9e4f30a1093a4514be1a7ccd33b71a46 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/9e4f30a1093a4514be1a7ccd33b71a46 2024-12-02T21:15:44,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/37906e1ab4ff4bba912aa868bdd8969c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/37906e1ab4ff4bba912aa868bdd8969c 2024-12-02T21:15:44,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/23d6a6a62a2743b78d12fd734aa57d7f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/23d6a6a62a2743b78d12fd734aa57d7f 2024-12-02T21:15:44,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e3f02059ccc945858094d5729e614bfd to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e3f02059ccc945858094d5729e614bfd 2024-12-02T21:15:44,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/ea90d3ac15e24839a22c6c98bf51c564 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/ea90d3ac15e24839a22c6c98bf51c564 2024-12-02T21:15:44,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b8871e85a30c49ad95f1d51a516eff1e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/b8871e85a30c49ad95f1d51a516eff1e 2024-12-02T21:15:44,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/41c4e0506b324853bd1da1095bbfe7da to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/41c4e0506b324853bd1da1095bbfe7da 2024-12-02T21:15:44,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e75de9370d6049aea4bfe4a4c289688a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/e75de9370d6049aea4bfe4a4c289688a 2024-12-02T21:15:44,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d92e74d3f90c4f1482461fe4e05c90ac to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/d92e74d3f90c4f1482461fe4e05c90ac 2024-12-02T21:15:44,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/44baddbc5f9d4fc8af3c311a617678bc to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/44baddbc5f9d4fc8af3c311a617678bc 2024-12-02T21:15:44,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1ac7532e2094498783be51553ab475ad to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/1ac7532e2094498783be51553ab475ad 2024-12-02T21:15:44,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/7dca620a69804aa39e9b7e132748bcc9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/7dca620a69804aa39e9b7e132748bcc9 2024-12-02T21:15:44,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/86d278e3134f47819b643fefd9576595 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/86d278e3134f47819b643fefd9576595 2024-12-02T21:15:44,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/02548343f2f74a5c822c43cf04f0e82a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/02548343f2f74a5c822c43cf04f0e82a 2024-12-02T21:15:44,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c3b848742ef343c581050effefd1f353 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/c3b848742ef343c581050effefd1f353 2024-12-02T21:15:44,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/f9f2c7356a0d48e49ff6bdd592745c1e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/f9f2c7356a0d48e49ff6bdd592745c1e 2024-12-02T21:15:44,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/47d2d5a4f4ab4c9db75d23585dfa3ac7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/47d2d5a4f4ab4c9db75d23585dfa3ac7 2024-12-02T21:15:44,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/318814ad6aee42c3b2817d0b6492dd05 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/318814ad6aee42c3b2817d0b6492dd05 2024-12-02T21:15:44,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/aed5edfcf2dd4b7a842a641c644e8f8f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/aed5edfcf2dd4b7a842a641c644e8f8f 2024-12-02T21:15:44,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/84f3ec3d00c849a289cfa08d3a4a647a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/84f3ec3d00c849a289cfa08d3a4a647a 2024-12-02T21:15:44,734 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/recovered.edits/660.seqid, newMaxSeqId=660, maxSeqId=1 2024-12-02T21:15:44,735 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. 2024-12-02T21:15:44,735 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 8a7bf3289fb638014baa7fb12fa0d82c: Waiting for close lock at 1733174143280Running coprocessor pre-close hooks at 1733174143280Disabling compacts and flushes for region at 1733174143280Disabling writes for close at 1733174143281 (+1 ms)Obtaining lock to block concurrent updates at 1733174143281Preparing flush snapshotting stores in 8a7bf3289fb638014baa7fb12fa0d82c at 1733174143281Finished memstore snapshotting TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733174143281Flushing stores of TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c. at 1733174143282 (+1 ms)Flushing 8a7bf3289fb638014baa7fb12fa0d82c/A: creating writer at 1733174143282Flushing 8a7bf3289fb638014baa7fb12fa0d82c/A: appending metadata at 1733174143285 (+3 ms)Flushing 8a7bf3289fb638014baa7fb12fa0d82c/A: closing flushed file at 1733174143285Flushing 8a7bf3289fb638014baa7fb12fa0d82c/B: creating writer at 1733174143697 (+412 ms)Flushing 8a7bf3289fb638014baa7fb12fa0d82c/B: appending metadata at 1733174143702 (+5 ms)Flushing 8a7bf3289fb638014baa7fb12fa0d82c/B: closing flushed file at 1733174143702Flushing 8a7bf3289fb638014baa7fb12fa0d82c/C: creating writer at 1733174144117 (+415 ms)Flushing 8a7bf3289fb638014baa7fb12fa0d82c/C: appending metadata at 1733174144122 (+5 ms)Flushing 8a7bf3289fb638014baa7fb12fa0d82c/C: closing flushed file at 1733174144122Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16d9fce1: reopening flushed file at 1733174144543 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39727d10: reopening flushed file at 1733174144551 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50dc4817: reopening flushed file at 1733174144556 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8a7bf3289fb638014baa7fb12fa0d82c in 1281ms, sequenceid=657, compaction requested=true at 1733174144562 (+6 ms)Writing region close event to WAL at 1733174144730 (+168 ms)Running coprocessor post-close hooks at 1733174144735 (+5 ms)Closed at 1733174144735 2024-12-02T21:15:44,738 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:44,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=8a7bf3289fb638014baa7fb12fa0d82c, regionState=CLOSED 2024-12-02T21:15:44,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:15:44,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-02T21:15:44,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a7bf3289fb638014baa7fb12fa0d82c, server=87c3fdb6c570,35563,1733174113859 in 1.6190 sec 2024-12-02T21:15:44,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-02T21:15:44,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a7bf3289fb638014baa7fb12fa0d82c, UNASSIGN in 1.6270 sec 2024-12-02T21:15:44,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-02T21:15:44,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6340 sec 2024-12-02T21:15:44,748 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174144748"}]},"ts":"1733174144748"} 2024-12-02T21:15:44,750 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T21:15:44,750 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T21:15:44,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6480 sec 2024-12-02T21:15:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T21:15:45,260 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:45,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T21:15:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:45,271 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-02T21:15:45,273 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:45,279 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:45,283 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/recovered.edits] 2024-12-02T21:15:45,286 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/141e4311f23e40789db5ed06886fd126 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/141e4311f23e40789db5ed06886fd126 2024-12-02T21:15:45,288 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4afcb43141364ebaadd1a89638e87cc3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/4afcb43141364ebaadd1a89638e87cc3 2024-12-02T21:15:45,289 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/6c553d7ca59c49518d3eefbc5a1e2aef to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/6c553d7ca59c49518d3eefbc5a1e2aef 2024-12-02T21:15:45,291 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbb94c20e9c446b5b5d8781b855ace62 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/A/fbb94c20e9c446b5b5d8781b855ace62 2024-12-02T21:15:45,294 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1f41d40730d04c078564e05a1d65c88b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/1f41d40730d04c078564e05a1d65c88b 2024-12-02T21:15:45,296 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/44c8b670f49741bbbc159cd4b3fe7b1d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/44c8b670f49741bbbc159cd4b3fe7b1d 2024-12-02T21:15:45,297 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/981563a0ca514776bd1f492f31bd656e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/981563a0ca514776bd1f492f31bd656e 2024-12-02T21:15:45,299 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef8cc04df9ce41db9b27ef56963d1723 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/B/ef8cc04df9ce41db9b27ef56963d1723 2024-12-02T21:15:45,301 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08b93cc06a8545fcac892de533a5e187 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/08b93cc06a8545fcac892de533a5e187 2024-12-02T21:15:45,303 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/30ed97e54052434cb82b5a4f7f66c5f2 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/30ed97e54052434cb82b5a4f7f66c5f2 2024-12-02T21:15:45,304 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3959456a07994ae3a87275f4e33638db to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/3959456a07994ae3a87275f4e33638db 2024-12-02T21:15:45,305 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/a3737ca9bdcb48769a9c8f2dce0b9f2d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/C/a3737ca9bdcb48769a9c8f2dce0b9f2d 2024-12-02T21:15:45,308 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/recovered.edits/660.seqid to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c/recovered.edits/660.seqid 2024-12-02T21:15:45,309 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/8a7bf3289fb638014baa7fb12fa0d82c 2024-12-02T21:15:45,309 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T21:15:45,314 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:45,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-02T21:15:45,323 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T21:15:45,351 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T21:15:45,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:45,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T21:15:45,353 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733174145353"}]},"ts":"9223372036854775807"} 2024-12-02T21:15:45,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T21:15:45,358 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8a7bf3289fb638014baa7fb12fa0d82c, NAME => 'TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T21:15:45,358 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T21:15:45,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733174145358"}]},"ts":"9223372036854775807"} 2024-12-02T21:15:45,361 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T21:15:45,362 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:45,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 97 msec 2024-12-02T21:15:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-02T21:15:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T21:15:45,380 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:45,392 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;87c3fdb6c570:35563-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1636387191_22 at /127.0.0.1:45414 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=565 (was 545) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2766 (was 2745) - AvailableMemoryMB LEAK? - 2024-12-02T21:15:45,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=565, ProcessCount=11, AvailableMemoryMB=2761 2024-12-02T21:15:45,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T21:15:45,406 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:15:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:45,409 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T21:15:45,409 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:45,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-12-02T21:15:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T21:15:45,410 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T21:15:45,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741962_1138 (size=1061) 2024-12-02T21:15:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T21:15:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T21:15:45,819 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0 2024-12-02T21:15:45,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741963_1139 (size=53) 2024-12-02T21:15:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T21:15:46,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:15:46,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ab928e240d6dc631659396e2f2d0cdda, disabling compactions & flushes 2024-12-02T21:15:46,227 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:46,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:46,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. after waiting 0 ms 2024-12-02T21:15:46,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:46,227 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:46,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ab928e240d6dc631659396e2f2d0cdda: Waiting for close lock at 1733174146227Disabling compacts and flushes for region at 1733174146227Disabling writes for close at 1733174146227Writing region close event to WAL at 1733174146227Closed at 1733174146227 2024-12-02T21:15:46,230 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T21:15:46,231 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733174146230"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174146230"}]},"ts":"1733174146230"} 2024-12-02T21:15:46,233 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T21:15:46,235 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T21:15:46,235 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174146235"}]},"ts":"1733174146235"} 2024-12-02T21:15:46,237 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T21:15:46,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, ASSIGN}] 2024-12-02T21:15:46,239 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, ASSIGN 2024-12-02T21:15:46,240 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, ASSIGN; state=OFFLINE, location=87c3fdb6c570,35563,1733174113859; forceNewPlan=false, retain=false 2024-12-02T21:15:46,390 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ab928e240d6dc631659396e2f2d0cdda, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:46,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, ASSIGN because future has completed 2024-12-02T21:15:46,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:15:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T21:15:46,554 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:46,555 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:15:46,555 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,556 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:15:46,556 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,556 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,559 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,560 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:15:46,561 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab928e240d6dc631659396e2f2d0cdda columnFamilyName A 2024-12-02T21:15:46,562 DEBUG [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:46,562 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(327): Store=ab928e240d6dc631659396e2f2d0cdda/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:46,563 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,564 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:15:46,564 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab928e240d6dc631659396e2f2d0cdda columnFamilyName B 2024-12-02T21:15:46,564 DEBUG [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:46,565 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(327): Store=ab928e240d6dc631659396e2f2d0cdda/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:46,565 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,566 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:15:46,567 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab928e240d6dc631659396e2f2d0cdda columnFamilyName C 2024-12-02T21:15:46,567 DEBUG [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:46,567 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(327): Store=ab928e240d6dc631659396e2f2d0cdda/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:46,567 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:46,568 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,568 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,569 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,570 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,570 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,570 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:15:46,572 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,574 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:15:46,574 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened ab928e240d6dc631659396e2f2d0cdda; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65739805, jitterRate=-0.02040056884288788}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:15:46,574 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:46,575 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for ab928e240d6dc631659396e2f2d0cdda: Running coprocessor pre-open hook at 1733174146556Writing region info on filesystem at 1733174146556Initializing all the Stores at 1733174146558 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174146558Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174146559 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174146559Cleaning up temporary data from old regions at 1733174146570 (+11 ms)Running coprocessor post-open hooks at 1733174146574 (+4 ms)Region opened successfully at 1733174146575 (+1 ms) 2024-12-02T21:15:46,576 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., pid=36, masterSystemTime=1733174146547 2024-12-02T21:15:46,578 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:46,578 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:46,579 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ab928e240d6dc631659396e2f2d0cdda, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:46,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:15:46,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-02T21:15:46,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 in 187 msec 2024-12-02T21:15:46,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-02T21:15:46,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, ASSIGN in 347 msec 2024-12-02T21:15:46,587 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T21:15:46,587 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174146587"}]},"ts":"1733174146587"} 2024-12-02T21:15:46,589 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T21:15:46,591 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T21:15:46,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-12-02T21:15:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T21:15:47,569 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:47,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:47,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:47,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:47,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:47,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:47,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cf34956, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:47,579 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ab66bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:47,580 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:47,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:47,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:47,585 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T21:15:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 87c3fdb6c570,39577,1733174113031 2024-12-02T21:15:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3af118e7 2024-12-02T21:15:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T21:15:47,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T21:15:47,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T21:15:47,593 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T21:15:47,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741964_1140 (size=1097) 2024-12-02T21:15:48,012 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T21:15:48,012 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T21:15:48,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T21:15:48,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, REOPEN/MOVE}] 2024-12-02T21:15:48,026 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, REOPEN/MOVE 2024-12-02T21:15:48,027 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ab928e240d6dc631659396e2f2d0cdda, regionState=CLOSING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, REOPEN/MOVE because future has completed 2024-12-02T21:15:48,029 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:15:48,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:15:48,182 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,182 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:15:48,183 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing ab928e240d6dc631659396e2f2d0cdda, disabling compactions & flushes 2024-12-02T21:15:48,183 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,183 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,183 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. after waiting 0 ms 2024-12-02T21:15:48,183 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,190 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:15:48,191 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,192 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for ab928e240d6dc631659396e2f2d0cdda: Waiting for close lock at 1733174148183Running coprocessor pre-close hooks at 1733174148183Disabling compacts and flushes for region at 1733174148183Disabling writes for close at 1733174148183Writing region close event to WAL at 1733174148184 (+1 ms)Running coprocessor post-close hooks at 1733174148191 (+7 ms)Closed at 1733174148191 2024-12-02T21:15:48,192 WARN [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: ab928e240d6dc631659396e2f2d0cdda to self. 2024-12-02T21:15:48,194 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,195 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ab928e240d6dc631659396e2f2d0cdda, regionState=CLOSED 2024-12-02T21:15:48,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:15:48,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-02T21:15:48,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 in 169 msec 2024-12-02T21:15:48,201 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, REOPEN/MOVE; state=CLOSED, location=87c3fdb6c570,35563,1733174113859; forceNewPlan=false, retain=true 2024-12-02T21:15:48,352 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ab928e240d6dc631659396e2f2d0cdda, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, REOPEN/MOVE because future has completed 2024-12-02T21:15:48,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:15:48,571 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,571 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:15:48,572 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,573 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:15:48,573 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,573 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,577 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,579 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:15:48,585 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab928e240d6dc631659396e2f2d0cdda columnFamilyName A 2024-12-02T21:15:48,586 DEBUG [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:48,587 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(327): Store=ab928e240d6dc631659396e2f2d0cdda/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:48,587 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,588 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:15:48,588 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab928e240d6dc631659396e2f2d0cdda columnFamilyName B 2024-12-02T21:15:48,588 DEBUG [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:48,589 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(327): Store=ab928e240d6dc631659396e2f2d0cdda/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:48,589 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,589 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:15:48,589 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab928e240d6dc631659396e2f2d0cdda columnFamilyName C 2024-12-02T21:15:48,590 DEBUG [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:48,590 INFO [StoreOpener-ab928e240d6dc631659396e2f2d0cdda-1 {}] regionserver.HStore(327): Store=ab928e240d6dc631659396e2f2d0cdda/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:15:48,590 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,590 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,591 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,592 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,593 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,593 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,593 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:15:48,594 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,595 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened ab928e240d6dc631659396e2f2d0cdda; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66794192, jitterRate=-0.00468897819519043}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:15:48,595 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,597 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for ab928e240d6dc631659396e2f2d0cdda: Running coprocessor pre-open hook at 1733174148573Writing region info on filesystem at 1733174148573Initializing all the Stores at 1733174148575 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174148575Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174148575Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174148575Cleaning up temporary data from old regions at 1733174148593 (+18 ms)Running coprocessor post-open hooks at 1733174148595 (+2 ms)Region opened successfully at 1733174148597 (+2 ms) 2024-12-02T21:15:48,598 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., pid=41, masterSystemTime=1733174148505 2024-12-02T21:15:48,599 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,600 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,600 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ab928e240d6dc631659396e2f2d0cdda, regionState=OPEN, openSeqNum=5, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:15:48,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-12-02T21:15:48,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 in 249 msec 2024-12-02T21:15:48,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-02T21:15:48,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, REOPEN/MOVE in 579 msec 2024-12-02T21:15:48,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-02T21:15:48,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 589 msec 2024-12-02T21:15:48,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0150 sec 2024-12-02T21:15:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-02T21:15:48,614 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:48,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:48,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:48,616 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f01562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:48,618 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a33850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:48,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:48,621 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1713fe00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:48,622 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a69afdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:48,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:48,625 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c031889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:48,627 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25a17fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:48,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:48,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fea2e90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:48,631 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@365793e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:48,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:48,634 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3404912b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:48,636 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f64517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:48,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:48,638 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f77207f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:48,640 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e1badaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:48,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:48,642 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d3d65be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:48,644 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc1f805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:48,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:48,646 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a6e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:48,647 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e5bca7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:15:48,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:15:48,649 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:15:48,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:15:48,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:15:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511a0f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:15:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:15:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:15:48,651 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:15:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dfab01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:15:48,652 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:48,652 DEBUG [Thread-704 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:48,652 DEBUG [Thread-702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:48,652 DEBUG [Thread-708 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:48,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:48,652 DEBUG [Thread-693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:48,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:48,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-02T21:15:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T21:15:48,655 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:48,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:48,656 DEBUG [Thread-706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:48,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:48,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:48,656 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:48,657 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:48,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:48,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:48,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:48,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:48,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:48,661 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:48,661 DEBUG [Thread-691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:15:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:15:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:15:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:48,662 DEBUG [Thread-691 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:15:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:48,663 DEBUG [Thread-691 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:15:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:15:48,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:15:48,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:15:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:15:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:15:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:15:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:15:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:15:48,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:15:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:48,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d7c8e7b7d26540c0a132709680cba646_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174148668/Put/seqid=0 2024-12-02T21:15:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733174148723, max timestamp=1733174148751Num uniques 859; 2024-12-02T21:15:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733174148723, max timestamp=1733174148751Num uniques 859; 2024-12-02T21:15:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733174148723, max timestamp=1733174148751Num uniques 859; 2024-12-02T21:15:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T21:15:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T21:15:48,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T21:15:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T21:15:48,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741965_1141 (size=14594) 2024-12-02T21:15:48,763 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:48,769 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d7c8e7b7d26540c0a132709680cba646_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d7c8e7b7d26540c0a132709680cba646_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:48,771 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/e29d23c48e054cd18070a4ca199cf928, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:48,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/e29d23c48e054cd18070a4ca199cf928 is 175, key is test_row_0/A:col10/1733174148668/Put/seqid=0 2024-12-02T21:15:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174208785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174208785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174208788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741966_1142 (size=39549) 2024-12-02T21:15:48,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174208792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174208793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,805 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/e29d23c48e054cd18070a4ca199cf928 2024-12-02T21:15:48,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T21:15:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:48,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/7a789c836f6b40d586b0b5cbf81954d5 is 50, key is test_row_0/B:col10/1733174148668/Put/seqid=0 2024-12-02T21:15:48,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741967_1143 (size=12001) 2024-12-02T21:15:48,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/7a789c836f6b40d586b0b5cbf81954d5 2024-12-02T21:15:48,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174208899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174208899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174208899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174208910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174208910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:48,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/ed0acde0909b4114ace92cf72de52e6b is 50, key is test_row_0/C:col10/1733174148668/Put/seqid=0 2024-12-02T21:15:48,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T21:15:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:48,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T21:15:48,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741968_1144 (size=12001) 2024-12-02T21:15:48,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/ed0acde0909b4114ace92cf72de52e6b 2024-12-02T21:15:49,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/e29d23c48e054cd18070a4ca199cf928 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e29d23c48e054cd18070a4ca199cf928 2024-12-02T21:15:49,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e29d23c48e054cd18070a4ca199cf928, entries=200, sequenceid=14, filesize=38.6 K 2024-12-02T21:15:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/7a789c836f6b40d586b0b5cbf81954d5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7a789c836f6b40d586b0b5cbf81954d5 2024-12-02T21:15:49,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7a789c836f6b40d586b0b5cbf81954d5, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T21:15:49,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/ed0acde0909b4114ace92cf72de52e6b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ed0acde0909b4114ace92cf72de52e6b 2024-12-02T21:15:49,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ed0acde0909b4114ace92cf72de52e6b, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T21:15:49,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for ab928e240d6dc631659396e2f2d0cdda in 376ms, sequenceid=14, compaction requested=false 2024-12-02T21:15:49,055 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T21:15:49,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:49,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-12-02T21:15:49,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:49,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:49,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:49,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:49,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:49,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:49,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T21:15:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:49,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:49,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:49,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a2b776ff6e574b5e9a3f470419f7c644_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174148776/Put/seqid=0 2024-12-02T21:15:49,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174209129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174209130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174209133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174209143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174209140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741969_1145 (size=17034) 2024-12-02T21:15:49,170 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a2b776ff6e574b5e9a3f470419f7c644_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a2b776ff6e574b5e9a3f470419f7c644_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:49,183 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/1f7eae28b3ef4edb8e9baf4d4c5f8594, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:49,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/1f7eae28b3ef4edb8e9baf4d4c5f8594 is 175, key is test_row_0/A:col10/1733174148776/Put/seqid=0 2024-12-02T21:15:49,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741970_1146 (size=48139) 2024-12-02T21:15:49,214 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/1f7eae28b3ef4edb8e9baf4d4c5f8594 2024-12-02T21:15:49,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/876a8b00117e4e3d8b1c57b8858341e7 is 50, key is test_row_0/B:col10/1733174148776/Put/seqid=0 2024-12-02T21:15:49,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741971_1147 (size=12001) 2024-12-02T21:15:49,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174209250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T21:15:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174209250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:49,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/876a8b00117e4e3d8b1c57b8858341e7 2024-12-02T21:15:49,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174209276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174209276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T21:15:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/3307da366a9241d48d45be3d0ec0027e is 50, key is test_row_0/C:col10/1733174148776/Put/seqid=0 2024-12-02T21:15:49,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741972_1148 (size=12001) 2024-12-02T21:15:49,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/3307da366a9241d48d45be3d0ec0027e 2024-12-02T21:15:49,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/1f7eae28b3ef4edb8e9baf4d4c5f8594 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1f7eae28b3ef4edb8e9baf4d4c5f8594 2024-12-02T21:15:49,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1f7eae28b3ef4edb8e9baf4d4c5f8594, entries=250, sequenceid=51, filesize=47.0 K 2024-12-02T21:15:49,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/876a8b00117e4e3d8b1c57b8858341e7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/876a8b00117e4e3d8b1c57b8858341e7 2024-12-02T21:15:49,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/876a8b00117e4e3d8b1c57b8858341e7, entries=150, sequenceid=51, filesize=11.7 K 2024-12-02T21:15:49,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/3307da366a9241d48d45be3d0ec0027e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/3307da366a9241d48d45be3d0ec0027e 2024-12-02T21:15:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/3307da366a9241d48d45be3d0ec0027e, entries=150, sequenceid=51, filesize=11.7 K 2024-12-02T21:15:49,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=33.54 KB/34350 for ab928e240d6dc631659396e2f2d0cdda in 239ms, sequenceid=51, compaction requested=false 2024-12-02T21:15:49,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T21:15:49,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,430 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T21:15:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024484934fce81499196432e1dd97373a6_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_1/A:col10/1733174149127/Put/seqid=0 2024-12-02T21:15:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741973_1149 (size=9714) 2024-12-02T21:15:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:49,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733174149504, max timestamp=1733174149526Num uniques 859; 2024-12-02T21:15:49,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:49,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733174149504, max timestamp=1733174149526Num uniques 859; 2024-12-02T21:15:49,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:49,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733174149504, max timestamp=1733174149526Num uniques 859; 2024-12-02T21:15:49,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:49,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174149569, max timestamp=1733174149577Num uniques 1068; 2024-12-02T21:15:49,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174149569, max timestamp=1733174149577Num uniques 1068; 2024-12-02T21:15:49,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-02T21:15:49,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-02T21:15:49,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174149569, max timestamp=1733174149577Num uniques 1068; 2024-12-02T21:15:49,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-02T21:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14963, totalHeapSize=34248, min timestamp=1733174149607, max timestamp=1733174149619Num uniques 1077; 2024-12-02T21:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14964, totalHeapSize=34248, min timestamp=1733174149607, max timestamp=1733174149619Num uniques 1077; 2024-12-02T21:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49313, new segment data size=14963, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-02T21:15:49,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49314, new segment data size=14964, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-02T21:15:49,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733174149607, max timestamp=1733174149618Num uniques 1077; 2024-12-02T21:15:49,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-02T21:15:49,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=436, cellSize=19966, totalHeapSize=45584, min timestamp=1733174149633, max timestamp=1733174149649Num uniques 1186; 2024-12-02T21:15:49,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=19966, suffix heap size=151312,new segment heap size=45584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=436 2024-12-02T21:15:49,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733174149646, max timestamp=1733174149649Num uniques 1186; 2024-12-02T21:15:49,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733174149646, max timestamp=1733174149649Num uniques 1186; 2024-12-02T21:15:49,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T21:15:49,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T21:15:49,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733174149665, max timestamp=1733174149684Num uniques 1295; 2024-12-02T21:15:49,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=59310, new segment data size=18090, suffix heap size=162648,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1295, new segment cells count=395 2024-12-02T21:15:49,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733174149665, max timestamp=1733174149684Num uniques 1045; 2024-12-02T21:15:49,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=18090, suffix heap size=136648,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=395 2024-12-02T21:15:49,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733174149665, max timestamp=1733174149684Num uniques 1045; 2024-12-02T21:15:49,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=18090, suffix heap size=136648,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=395 2024-12-02T21:15:49,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174209719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174209721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174209723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174209726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174209733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T21:15:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174209839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174209839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174209840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174209840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:49,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174209849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:49,858 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024484934fce81499196432e1dd97373a6_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024484934fce81499196432e1dd97373a6_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:49,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/6ede9e5e941c45dc8043915891919462, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:49,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/6ede9e5e941c45dc8043915891919462 is 175, key is test_row_1/A:col10/1733174149127/Put/seqid=0 2024-12-02T21:15:49,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741974_1150 (size=22361) 2024-12-02T21:15:49,883 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/6ede9e5e941c45dc8043915891919462 2024-12-02T21:15:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/4cedd3beeb3d4f0597757eb94b286ead is 50, key is test_row_1/B:col10/1733174149127/Put/seqid=0 2024-12-02T21:15:49,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741975_1151 (size=9657) 2024-12-02T21:15:50,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174210049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174210049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174210050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174210050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174210059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,299 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/4cedd3beeb3d4f0597757eb94b286ead 2024-12-02T21:15:50,311 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T21:15:50,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/5cd82b9718864275b4162f1ddeb32108 is 50, key is test_row_1/C:col10/1733174149127/Put/seqid=0 2024-12-02T21:15:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174210359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174210359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174210369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174210369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174210379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:50,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741976_1152 (size=9657) 2024-12-02T21:15:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T21:15:50,815 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/5cd82b9718864275b4162f1ddeb32108 2024-12-02T21:15:50,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/6ede9e5e941c45dc8043915891919462 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/6ede9e5e941c45dc8043915891919462 2024-12-02T21:15:50,831 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/6ede9e5e941c45dc8043915891919462, entries=100, sequenceid=59, filesize=21.8 K 2024-12-02T21:15:50,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/4cedd3beeb3d4f0597757eb94b286ead as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4cedd3beeb3d4f0597757eb94b286ead 2024-12-02T21:15:50,844 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4cedd3beeb3d4f0597757eb94b286ead, entries=100, sequenceid=59, filesize=9.4 K 2024-12-02T21:15:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/5cd82b9718864275b4162f1ddeb32108 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/5cd82b9718864275b4162f1ddeb32108 2024-12-02T21:15:50,852 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/5cd82b9718864275b4162f1ddeb32108, entries=100, sequenceid=59, filesize=9.4 K 2024-12-02T21:15:50,853 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for ab928e240d6dc631659396e2f2d0cdda in 1424ms, sequenceid=59, compaction requested=true 2024-12-02T21:15:50,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:50,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:50,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-02T21:15:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-02T21:15:50,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-02T21:15:50,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1980 sec 2024-12-02T21:15:50,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.2070 sec 2024-12-02T21:15:50,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=167.72 KB heapSize=462.94 KB 2024-12-02T21:15:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:50,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733174149705, max timestamp=1733174150885Num uniques 1254; 2024-12-02T21:15:50,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733174149705, max timestamp=1733174150885Num uniques 1254; 2024-12-02T21:15:50,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-12-02T21:15:50,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-12-02T21:15:50,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733174149705, max timestamp=1733174150885Num uniques 1254; 2024-12-02T21:15:50,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-12-02T21:15:50,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:50,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:50,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:50,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:50,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:50,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:50,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202631cb20fa5aa4e03b173f4d4b8b17d71_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174150883/Put/seqid=0 2024-12-02T21:15:50,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741977_1153 (size=14794) 2024-12-02T21:15:50,968 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:50,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202631cb20fa5aa4e03b173f4d4b8b17d71_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202631cb20fa5aa4e03b173f4d4b8b17d71_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:50,995 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/b594264600ae47d4b999e05fdb08cf7b, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:50,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/b594264600ae47d4b999e05fdb08cf7b is 175, key is test_row_0/A:col10/1733174150883/Put/seqid=0 2024-12-02T21:15:51,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174150938, max timestamp=1733174151011Num uniques 859; 2024-12-02T21:15:51,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:51,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1733174150938, max timestamp=1733174151011Num uniques 859; 2024-12-02T21:15:51,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8240, suffix heap size=117064,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=180 2024-12-02T21:15:51,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174150938, max timestamp=1733174151011Num uniques 859; 2024-12-02T21:15:51,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:51,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741978_1154 (size=39749) 2024-12-02T21:15:51,045 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/b594264600ae47d4b999e05fdb08cf7b 2024-12-02T21:15:51,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/c204d08751af41bc932d06fc2b3e36f1 is 50, key is test_row_0/B:col10/1733174150883/Put/seqid=0 2024-12-02T21:15:51,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174151073, max timestamp=1733174151105Num uniques 1068; 2024-12-02T21:15:51,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-02T21:15:51,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174151073, max timestamp=1733174151108Num uniques 1018; 2024-12-02T21:15:51,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174151073, max timestamp=1733174151108Num uniques 1039; 2024-12-02T21:15:51,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-02T21:15:51,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=9978, suffix heap size=136024,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=218 2024-12-02T21:15:51,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741979_1155 (size=12151) 2024-12-02T21:15:51,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/c204d08751af41bc932d06fc2b3e36f1 2024-12-02T21:15:51,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/83bbcd7c154f46bbbea67d5a4513e141 is 50, key is test_row_0/C:col10/1733174150883/Put/seqid=0 2024-12-02T21:15:51,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1733174151138, max timestamp=1733174151177Num uniques 1077; 2024-12-02T21:15:51,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=19552, suffix heap size=139976,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=427 2024-12-02T21:15:51,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1733174151138, max timestamp=1733174151177Num uniques 1077; 2024-12-02T21:15:51,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1733174151138, max timestamp=1733174151177Num uniques 1077; 2024-12-02T21:15:51,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=19552, suffix heap size=139976,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=427 2024-12-02T21:15:51,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=19552, suffix heap size=139976,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=427 2024-12-02T21:15:51,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741980_1156 (size=12151) 2024-12-02T21:15:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174211224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174211232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174211234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174211235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174211253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174211349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174211349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174211349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174211349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174211359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174211559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174211570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174211570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174211570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174211570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:51,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/83bbcd7c154f46bbbea67d5a4513e141 2024-12-02T21:15:51,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/b594264600ae47d4b999e05fdb08cf7b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/b594264600ae47d4b999e05fdb08cf7b 2024-12-02T21:15:51,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/b594264600ae47d4b999e05fdb08cf7b, entries=200, sequenceid=166, filesize=38.8 K 2024-12-02T21:15:51,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/c204d08751af41bc932d06fc2b3e36f1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/c204d08751af41bc932d06fc2b3e36f1 2024-12-02T21:15:51,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/c204d08751af41bc932d06fc2b3e36f1, entries=150, sequenceid=166, filesize=11.9 K 2024-12-02T21:15:51,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/83bbcd7c154f46bbbea67d5a4513e141 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/83bbcd7c154f46bbbea67d5a4513e141 2024-12-02T21:15:51,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/83bbcd7c154f46bbbea67d5a4513e141, entries=150, sequenceid=166, filesize=11.9 K 2024-12-02T21:15:51,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.66 KB/68256, currentSize=167.72 KB/171750 for ab928e240d6dc631659396e2f2d0cdda in 751ms, sequenceid=166, compaction requested=true 2024-12-02T21:15:51,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:51,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:51,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:51,637 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:51,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:51,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:51,637 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:51,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:51,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:51,640 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149798 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:51,640 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:51,640 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/B is initiating minor compaction (all files) 2024-12-02T21:15:51,640 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/A is initiating minor compaction (all files) 2024-12-02T21:15:51,641 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/B in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:51,641 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/A in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:51,641 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e29d23c48e054cd18070a4ca199cf928, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1f7eae28b3ef4edb8e9baf4d4c5f8594, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/6ede9e5e941c45dc8043915891919462, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/b594264600ae47d4b999e05fdb08cf7b] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=146.3 K 2024-12-02T21:15:51,641 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7a789c836f6b40d586b0b5cbf81954d5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/876a8b00117e4e3d8b1c57b8858341e7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4cedd3beeb3d4f0597757eb94b286ead, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/c204d08751af41bc932d06fc2b3e36f1] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=44.7 K 2024-12-02T21:15:51,641 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:51,641 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e29d23c48e054cd18070a4ca199cf928, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1f7eae28b3ef4edb8e9baf4d4c5f8594, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/6ede9e5e941c45dc8043915891919462, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/b594264600ae47d4b999e05fdb08cf7b] 2024-12-02T21:15:51,642 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a789c836f6b40d586b0b5cbf81954d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733174148668 2024-12-02T21:15:51,642 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e29d23c48e054cd18070a4ca199cf928, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733174148668 2024-12-02T21:15:51,642 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 876a8b00117e4e3d8b1c57b8858341e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733174148776 2024-12-02T21:15:51,642 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f7eae28b3ef4edb8e9baf4d4c5f8594, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733174148776 2024-12-02T21:15:51,643 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cedd3beeb3d4f0597757eb94b286ead, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733174149127 2024-12-02T21:15:51,643 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ede9e5e941c45dc8043915891919462, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733174149127 2024-12-02T21:15:51,643 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting c204d08751af41bc932d06fc2b3e36f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733174149705 2024-12-02T21:15:51,643 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting b594264600ae47d4b999e05fdb08cf7b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733174149705 2024-12-02T21:15:51,661 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:51,663 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#B#compaction#137 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:51,664 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/b65b1fdbb47f4570a5c5a8051a4ab107 is 50, key is test_row_0/B:col10/1733174150883/Put/seqid=0 2024-12-02T21:15:51,674 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120210a56c1978e44728bbcea54d5e79ffd0_ab928e240d6dc631659396e2f2d0cdda store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:51,679 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120210a56c1978e44728bbcea54d5e79ffd0_ab928e240d6dc631659396e2f2d0cdda, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:51,679 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120210a56c1978e44728bbcea54d5e79ffd0_ab928e240d6dc631659396e2f2d0cdda because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:51,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741981_1157 (size=12289) 2024-12-02T21:15:51,748 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/b65b1fdbb47f4570a5c5a8051a4ab107 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b65b1fdbb47f4570a5c5a8051a4ab107 2024-12-02T21:15:51,756 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/B of ab928e240d6dc631659396e2f2d0cdda into b65b1fdbb47f4570a5c5a8051a4ab107(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:51,756 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:51,756 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/B, priority=12, startTime=1733174151637; duration=0sec 2024-12-02T21:15:51,756 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:51,756 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:B 2024-12-02T21:15:51,756 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:15:51,758 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:15:51,759 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/C is initiating minor compaction (all files) 2024-12-02T21:15:51,760 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/C in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:51,760 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ed0acde0909b4114ace92cf72de52e6b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/3307da366a9241d48d45be3d0ec0027e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/5cd82b9718864275b4162f1ddeb32108, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/83bbcd7c154f46bbbea67d5a4513e141] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=44.7 K 2024-12-02T21:15:51,761 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ed0acde0909b4114ace92cf72de52e6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733174148668 2024-12-02T21:15:51,761 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 3307da366a9241d48d45be3d0ec0027e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733174148776 2024-12-02T21:15:51,762 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd82b9718864275b4162f1ddeb32108, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733174149127 2024-12-02T21:15:51,763 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 83bbcd7c154f46bbbea67d5a4513e141, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733174149705 2024-12-02T21:15:51,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741982_1158 (size=4469) 2024-12-02T21:15:51,791 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#C#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:51,792 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/1d0dce0a1bee4921b1a1a9cbd5ffb903 is 50, key is test_row_0/C:col10/1733174150883/Put/seqid=0 2024-12-02T21:15:51,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741983_1159 (size=12289) 2024-12-02T21:15:51,835 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/1d0dce0a1bee4921b1a1a9cbd5ffb903 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/1d0dce0a1bee4921b1a1a9cbd5ffb903 2024-12-02T21:15:51,842 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/C of ab928e240d6dc631659396e2f2d0cdda into 1d0dce0a1bee4921b1a1a9cbd5ffb903(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:51,842 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:51,842 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/C, priority=12, startTime=1733174151637; duration=0sec 2024-12-02T21:15:51,842 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:51,843 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:C 2024-12-02T21:15:51,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733174151231, max timestamp=1733174151885Num uniques 1286; 2024-12-02T21:15:51,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=58896, new segment data size=8516, suffix heap size=161712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1286, new segment cells count=186 2024-12-02T21:15:51,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=125.23 KB heapSize=341.30 KB 2024-12-02T21:15:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:51,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733174151231, max timestamp=1733174151885Num uniques 1286; 2024-12-02T21:15:51,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=58896, new segment data size=8516, suffix heap size=161712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1286, new segment cells count=186 2024-12-02T21:15:51,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:51,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733174151231, max timestamp=1733174151885Num uniques 1286; 2024-12-02T21:15:51,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:51,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:51,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=58896, new segment data size=8516, suffix heap size=161712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1286, new segment cells count=186 2024-12-02T21:15:51,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:51,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:51,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:51,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020613088f8f5046e29dde38bc3e99d76d_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174151231/Put/seqid=0 2024-12-02T21:15:51,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174151924, max timestamp=1733174151938Num uniques 859; 2024-12-02T21:15:51,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174151924, max timestamp=1733174151938Num uniques 859; 2024-12-02T21:15:51,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174151924, max timestamp=1733174151938Num uniques 859; 2024-12-02T21:15:51,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:15:51,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:15:51,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:15:51,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741984_1160 (size=12304) 2024-12-02T21:15:51,969 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:51,976 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020613088f8f5046e29dde38bc3e99d76d_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020613088f8f5046e29dde38bc3e99d76d_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:51,978 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a3511caca2824287bd8964f9ce4bdff7, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:51,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a3511caca2824287bd8964f9ce4bdff7 is 175, key is test_row_0/A:col10/1733174151231/Put/seqid=0 2024-12-02T21:15:52,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741985_1161 (size=31105) 2024-12-02T21:15:52,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a3511caca2824287bd8964f9ce4bdff7 2024-12-02T21:15:52,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1733174152020, max timestamp=1733174152058Num uniques 1118; 2024-12-02T21:15:52,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-12-02T21:15:52,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1733174152020, max timestamp=1733174152058Num uniques 1118; 2024-12-02T21:15:52,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-12-02T21:15:52,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1733174152020, max timestamp=1733174152058Num uniques 1118; 2024-12-02T21:15:52,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-12-02T21:15:52,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/4ecf5a42d974454990dc0be335576c3e is 50, key is test_row_0/B:col10/1733174151231/Put/seqid=0 2024-12-02T21:15:52,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741986_1162 (size=12151) 2024-12-02T21:15:52,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174212137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733174152131, max timestamp=1733174152144Num uniques 1227; 2024-12-02T21:15:52,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=14972, suffix heap size=155576,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=327 2024-12-02T21:15:52,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733174152131, max timestamp=1733174152144Num uniques 1227; 2024-12-02T21:15:52,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=14972, suffix heap size=155576,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=327 2024-12-02T21:15:52,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733174152131, max timestamp=1733174152144Num uniques 1227; 2024-12-02T21:15:52,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=14972, suffix heap size=155576,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=327 2024-12-02T21:15:52,183 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:52,183 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/65ab8149700c4773b9d495ed62a6d394 is 175, key is test_row_0/A:col10/1733174150883/Put/seqid=0 2024-12-02T21:15:52,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741987_1163 (size=31243) 2024-12-02T21:15:52,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174212183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733174152174, max timestamp=1733174152188Num uniques 1186; 2024-12-02T21:15:52,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-12-02T21:15:52,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733174152174, max timestamp=1733174152188Num uniques 1186; 2024-12-02T21:15:52,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733174152174, max timestamp=1733174152188Num uniques 1186; 2024-12-02T21:15:52,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-12-02T21:15:52,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-12-02T21:15:52,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733174152207, max timestamp=1733174152223Num uniques 1095; 2024-12-02T21:15:52,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T21:15:52,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733174152207, max timestamp=1733174152223Num uniques 1095; 2024-12-02T21:15:52,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T21:15:52,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733174152207, max timestamp=1733174152223Num uniques 1095; 2024-12-02T21:15:52,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T21:15:52,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174212248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174212250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174212251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174212252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174212299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174212359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174212359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174212359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174212459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/4ecf5a42d974454990dc0be335576c3e 2024-12-02T21:15:52,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/a6a6a7c31a24425fa8b3839ca1a2ba97 is 50, key is test_row_0/C:col10/1733174151231/Put/seqid=0 2024-12-02T21:15:52,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741988_1164 (size=12151) 2024-12-02T21:15:52,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/a6a6a7c31a24425fa8b3839ca1a2ba97 2024-12-02T21:15:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174212509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a3511caca2824287bd8964f9ce4bdff7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a3511caca2824287bd8964f9ce4bdff7 2024-12-02T21:15:52,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a3511caca2824287bd8964f9ce4bdff7, entries=150, sequenceid=241, filesize=30.4 K 2024-12-02T21:15:52,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/4ecf5a42d974454990dc0be335576c3e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4ecf5a42d974454990dc0be335576c3e 2024-12-02T21:15:52,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4ecf5a42d974454990dc0be335576c3e, entries=150, sequenceid=241, filesize=11.9 K 2024-12-02T21:15:52,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/a6a6a7c31a24425fa8b3839ca1a2ba97 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/a6a6a7c31a24425fa8b3839ca1a2ba97 2024-12-02T21:15:52,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/a6a6a7c31a24425fa8b3839ca1a2ba97, entries=150, sequenceid=241, filesize=11.9 K 2024-12-02T21:15:52,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~83.58 KB/85584, currentSize=154.31 KB/158010 for ab928e240d6dc631659396e2f2d0cdda in 655ms, sequenceid=241, compaction requested=false 2024-12-02T21:15:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:52,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=167.72 KB heapSize=472.55 KB 2024-12-02T21:15:52,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:52,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T21:15:52,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:52,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T21:15:52,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:52,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T21:15:52,597 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/65ab8149700c4773b9d495ed62a6d394 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/65ab8149700c4773b9d495ed62a6d394 2024-12-02T21:15:52,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a7fbb6eeebe443af821dad3341a02326_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174152569/Put/seqid=0 2024-12-02T21:15:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174212595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174212597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,606 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/A of ab928e240d6dc631659396e2f2d0cdda into 65ab8149700c4773b9d495ed62a6d394(size=30.5 K), total size for store is 60.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:52,606 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,606 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/A, priority=12, startTime=1733174151637; duration=0sec 2024-12-02T21:15:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,606 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:52,606 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:A 2024-12-02T21:15:52,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174212603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741989_1165 (size=12454) 2024-12-02T21:15:52,618 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:52,624 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a7fbb6eeebe443af821dad3341a02326_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a7fbb6eeebe443af821dad3341a02326_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:52,625 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/39d0c42940514e879da51356e93c390f, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:52,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/39d0c42940514e879da51356e93c390f is 175, key is test_row_0/A:col10/1733174152569/Put/seqid=0 2024-12-02T21:15:52,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741990_1166 (size=31255) 2024-12-02T21:15:52,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/39d0c42940514e879da51356e93c390f 2024-12-02T21:15:52,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/4829332c41e6493abc73d131c3e52dd2 is 50, key is test_row_0/B:col10/1733174152569/Put/seqid=0 2024-12-02T21:15:52,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741991_1167 (size=12301) 2024-12-02T21:15:52,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174212708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174212708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174212719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174212768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T21:15:52,809 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:52,810 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-02T21:15:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T21:15:52,812 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:52,813 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:52,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:52,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174212819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T21:15:52,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174212919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174212919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174212928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:52,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T21:15:52,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:52,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:52,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:52,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:52,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/4829332c41e6493abc73d131c3e52dd2 2024-12-02T21:15:53,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/888a2feea273403d8ccd6aee1c745daf is 50, key is test_row_0/C:col10/1733174152569/Put/seqid=0 2024-12-02T21:15:53,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741992_1168 (size=12301) 2024-12-02T21:15:53,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T21:15:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T21:15:53,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174213229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:53,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174213229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:53,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174213239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:53,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T21:15:53,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:53,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174213278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:53,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174213329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:53,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T21:15:53,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:53,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T21:15:53,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/888a2feea273403d8ccd6aee1c745daf 2024-12-02T21:15:53,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/39d0c42940514e879da51356e93c390f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39d0c42940514e879da51356e93c390f 2024-12-02T21:15:53,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39d0c42940514e879da51356e93c390f, entries=150, sequenceid=349, filesize=30.5 K 2024-12-02T21:15:53,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/4829332c41e6493abc73d131c3e52dd2 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4829332c41e6493abc73d131c3e52dd2 2024-12-02T21:15:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4829332c41e6493abc73d131c3e52dd2, entries=150, sequenceid=349, filesize=12.0 K 2024-12-02T21:15:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/888a2feea273403d8ccd6aee1c745daf as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/888a2feea273403d8ccd6aee1c745daf 2024-12-02T21:15:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/888a2feea273403d8ccd6aee1c745daf, entries=150, sequenceid=349, filesize=12.0 K 2024-12-02T21:15:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~472.50 KB/483840, currentSize=13.42 KB/13740 for ab928e240d6dc631659396e2f2d0cdda in 983ms, sequenceid=349, compaction requested=true 2024-12-02T21:15:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:53,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:53,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:53,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:53,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T21:15:53,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:53,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T21:15:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,575 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:53,576 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,577 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:53,577 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36741 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:53,577 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/C is initiating minor compaction (all files) 2024-12-02T21:15:53,577 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/A is initiating minor compaction (all files) 2024-12-02T21:15:53,578 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/C in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,578 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/1d0dce0a1bee4921b1a1a9cbd5ffb903, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/a6a6a7c31a24425fa8b3839ca1a2ba97, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/888a2feea273403d8ccd6aee1c745daf] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=35.9 K 2024-12-02T21:15:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,578 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/A in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,578 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/65ab8149700c4773b9d495ed62a6d394, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a3511caca2824287bd8964f9ce4bdff7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39d0c42940514e879da51356e93c390f] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=91.4 K 2024-12-02T21:15:53,578 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,578 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/65ab8149700c4773b9d495ed62a6d394, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a3511caca2824287bd8964f9ce4bdff7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39d0c42940514e879da51356e93c390f] 2024-12-02T21:15:53,579 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d0dce0a1bee4921b1a1a9cbd5ffb903, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733174149705 2024-12-02T21:15:53,579 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ab8149700c4773b9d495ed62a6d394, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733174149705 2024-12-02T21:15:53,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T21:15:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:53,579 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-02T21:15:53,580 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting a6a6a7c31a24425fa8b3839ca1a2ba97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733174151231 2024-12-02T21:15:53,580 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3511caca2824287bd8964f9ce4bdff7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733174151231 2024-12-02T21:15:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,580 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 888a2feea273403d8ccd6aee1c745daf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733174152251 2024-12-02T21:15:53,581 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39d0c42940514e879da51356e93c390f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733174152251 2024-12-02T21:15:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,615 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,615 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/123dd432968645f58d675bc258ad0b0e is 50, key is test_row_0/C:col10/1733174152569/Put/seqid=0 2024-12-02T21:15:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,617 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027fcac0336c7e45bd8167cd8528fb2170_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174152593/Put/seqid=0 2024-12-02T21:15:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,637 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202037976c0599642e7a10f196050bfbda3_ab928e240d6dc631659396e2f2d0cdda store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,639 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202037976c0599642e7a10f196050bfbda3_ab928e240d6dc631659396e2f2d0cdda, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:53,639 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202037976c0599642e7a10f196050bfbda3_ab928e240d6dc631659396e2f2d0cdda because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741994_1170 (size=9914) 2024-12-02T21:15:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,662 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027fcac0336c7e45bd8167cd8528fb2170_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fcac0336c7e45bd8167cd8528fb2170_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8d1cdef2dcb34f03bc10f26d4204aaac, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8d1cdef2dcb34f03bc10f26d4204aaac is 175, key is test_row_0/A:col10/1733174152593/Put/seqid=0 2024-12-02T21:15:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741995_1171 (size=4469) 2024-12-02T21:15:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741993_1169 (size=12541) 2024-12-02T21:15:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741996_1172 (size=22561) 2024-12-02T21:15:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:53,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:53,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=759, cellSize=34755, totalHeapSize=79176, min timestamp=1733174153746, max timestamp=1733174153795Num uniques 859; 2024-12-02T21:15:53,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=34755, suffix heap size=117064,new segment heap size=79176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=759 2024-12-02T21:15:53,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174153788, max timestamp=1733174153795Num uniques 859; 2024-12-02T21:15:53,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:53,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174153788, max timestamp=1733174153795Num uniques 859; 2024-12-02T21:15:53,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:15:53,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174213828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:53,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174153814, max timestamp=1733174153831Num uniques 1068; 2024-12-02T21:15:53,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-02T21:15:53,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174153814, max timestamp=1733174153831Num uniques 1618; 2024-12-02T21:15:53,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=74098, new segment data size=9978, suffix heap size=196240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1618, new segment cells count=218 2024-12-02T21:15:53,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174153814, max timestamp=1733174153831Num uniques 1068; 2024-12-02T21:15:53,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-02T21:15:53,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733174153860, max timestamp=1733174153874Num uniques 1077; 2024-12-02T21:15:53,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-02T21:15:53,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733174153860, max timestamp=1733174153874Num uniques 1077; 2024-12-02T21:15:53,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733174153860, max timestamp=1733174153874Num uniques 1077; 2024-12-02T21:15:53,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-02T21:15:53,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-02T21:15:53,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733174153885, max timestamp=1733174153920Num uniques 1136; 2024-12-02T21:15:53,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733174153885, max timestamp=1733174153920Num uniques 1136; 2024-12-02T21:15:53,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-02T21:15:53,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-02T21:15:53,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733174153885, max timestamp=1733174153920Num uniques 1136; 2024-12-02T21:15:53,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-02T21:15:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T21:15:53,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733174153951, max timestamp=1733174153972Num uniques 1045; 2024-12-02T21:15:53,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-02T21:15:53,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733174153951, max timestamp=1733174153972Num uniques 1045; 2024-12-02T21:15:53,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-02T21:15:53,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733174153951, max timestamp=1733174153972Num uniques 1045; 2024-12-02T21:15:53,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-02T21:15:54,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733174154016, max timestamp=1733174154031Num uniques 1104; 2024-12-02T21:15:54,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733174154016, max timestamp=1733174154031Num uniques 1154; 2024-12-02T21:15:54,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-12-02T21:15:54,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-02T21:15:54,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733174154016, max timestamp=1733174154031Num uniques 1104; 2024-12-02T21:15:54,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-02T21:15:54,071 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:54,071 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/0c5005349f9b4580bcb55aff43739704 is 175, key is test_row_0/A:col10/1733174152569/Put/seqid=0 2024-12-02T21:15:54,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733174154052, max timestamp=1733174154068Num uniques 1113; 2024-12-02T21:15:54,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-12-02T21:15:54,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733174154052, max timestamp=1733174154068Num uniques 1113; 2024-12-02T21:15:54,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-12-02T21:15:54,075 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8d1cdef2dcb34f03bc10f26d4204aaac 2024-12-02T21:15:54,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733174154052, max timestamp=1733174154068Num uniques 1113; 2024-12-02T21:15:54,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-12-02T21:15:54,084 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/123dd432968645f58d675bc258ad0b0e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/123dd432968645f58d675bc258ad0b0e 2024-12-02T21:15:54,094 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/C of ab928e240d6dc631659396e2f2d0cdda into 123dd432968645f58d675bc258ad0b0e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:54,094 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:54,094 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/C, priority=13, startTime=1733174153575; duration=0sec 2024-12-02T21:15:54,094 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:54,094 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:C 2024-12-02T21:15:54,094 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:54,096 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36741 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:54,096 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/B is initiating minor compaction (all files) 2024-12-02T21:15:54,096 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/B in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:54,096 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b65b1fdbb47f4570a5c5a8051a4ab107, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4ecf5a42d974454990dc0be335576c3e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4829332c41e6493abc73d131c3e52dd2] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=35.9 K 2024-12-02T21:15:54,097 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b65b1fdbb47f4570a5c5a8051a4ab107, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733174149705 2024-12-02T21:15:54,097 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ecf5a42d974454990dc0be335576c3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733174151231 2024-12-02T21:15:54,098 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 4829332c41e6493abc73d131c3e52dd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733174152251 2024-12-02T21:15:54,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/23973d9992454b7d87a1b800aac92f76 is 50, key is test_row_0/B:col10/1733174152593/Put/seqid=0 2024-12-02T21:15:54,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733174154087, max timestamp=1733174154119Num uniques 1122; 2024-12-02T21:15:54,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733174154087, max timestamp=1733174154119Num uniques 1122; 2024-12-02T21:15:54,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-12-02T21:15:54,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-12-02T21:15:54,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733174154087, max timestamp=1733174154119Num uniques 1122; 2024-12-02T21:15:54,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-12-02T21:15:54,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741997_1173 (size=31495) 2024-12-02T21:15:54,134 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:54,135 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/78aa4cdb8b3b41a3b149b56edb83e2c5 is 50, key is test_row_0/B:col10/1733174152569/Put/seqid=0 2024-12-02T21:15:54,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741998_1174 (size=9857) 2024-12-02T21:15:54,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741999_1175 (size=12541) 2024-12-02T21:15:54,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733174154141, max timestamp=1733174154171Num uniques 1081; 2024-12-02T21:15:54,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733174154141, max timestamp=1733174154171Num uniques 1081; 2024-12-02T21:15:54,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733174154141, max timestamp=1733174154171Num uniques 1081; 2024-12-02T21:15:54,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-02T21:15:54,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-02T21:15:54,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-02T21:15:54,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733174154186, max timestamp=1733174154211Num uniques 1090; 2024-12-02T21:15:54,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=15570, suffix heap size=141328,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=340 2024-12-02T21:15:54,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733174154186, max timestamp=1733174154211Num uniques 1090; 2024-12-02T21:15:54,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=15570, suffix heap size=141328,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=340 2024-12-02T21:15:54,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733174154186, max timestamp=1733174154211Num uniques 1090; 2024-12-02T21:15:54,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=15570, suffix heap size=141328,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=340 2024-12-02T21:15:54,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733174154237, max timestamp=1733174154246Num uniques 1199; 2024-12-02T21:15:54,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=11404, suffix heap size=152664,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=249 2024-12-02T21:15:54,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733174154237, max timestamp=1733174154246Num uniques 1199; 2024-12-02T21:15:54,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=11404, suffix heap size=152664,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=249 2024-12-02T21:15:54,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733174154237, max timestamp=1733174154246Num uniques 1199; 2024-12-02T21:15:54,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=11404, suffix heap size=152664,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=249 2024-12-02T21:15:54,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16390, totalHeapSize=37472, min timestamp=1733174154264, max timestamp=1733174154280Num uniques 1108; 2024-12-02T21:15:54,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=16390, suffix heap size=143200,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=358 2024-12-02T21:15:54,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16390, totalHeapSize=37472, min timestamp=1733174154264, max timestamp=1733174154280Num uniques 1108; 2024-12-02T21:15:54,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=16390, suffix heap size=143200,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=358 2024-12-02T21:15:54,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733174154270, max timestamp=1733174154280Num uniques 1108; 2024-12-02T21:15:54,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-12-02T21:15:54,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733174154300, max timestamp=1733174154306Num uniques 1217; 2024-12-02T21:15:54,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=55732, new segment data size=12222, suffix heap size=154536,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1217, new segment cells count=267 2024-12-02T21:15:54,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733174154300, max timestamp=1733174154306Num uniques 1217; 2024-12-02T21:15:54,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733174154300, max timestamp=1733174154306Num uniques 1017; 2024-12-02T21:15:54,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55732, new segment data size=12222, suffix heap size=154536,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1217, new segment cells count=267 2024-12-02T21:15:54,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-12-02T21:15:54,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733174154326, max timestamp=1733174154335Num uniques 1126; 2024-12-02T21:15:54,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=8056, suffix heap size=145072,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=176 2024-12-02T21:15:54,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733174154326, max timestamp=1733174154335Num uniques 1126; 2024-12-02T21:15:54,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733174154326, max timestamp=1733174154335Num uniques 1126; 2024-12-02T21:15:54,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=8056, suffix heap size=145072,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=176 2024-12-02T21:15:54,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=8056, suffix heap size=145072,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=176 2024-12-02T21:15:54,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=385, cellSize=17630, totalHeapSize=40280, min timestamp=1733174154344, max timestamp=1733174154359Num uniques 1035; 2024-12-02T21:15:54,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=17630, suffix heap size=135608,new segment heap size=40280 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=385 2024-12-02T21:15:54,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733174154344, max timestamp=1733174154359Num uniques 1035; 2024-12-02T21:15:54,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-02T21:15:54,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=385, cellSize=17630, totalHeapSize=40280, min timestamp=1733174154344, max timestamp=1733174154359Num uniques 1035; 2024-12-02T21:15:54,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=17630, suffix heap size=135608,new segment heap size=40280 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=385 2024-12-02T21:15:54,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=444, cellSize=20334, totalHeapSize=46416, min timestamp=1733174154368, max timestamp=1733174154382Num uniques 1244; 2024-12-02T21:15:54,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=56974, new segment data size=20334, suffix heap size=157344,new segment heap size=46416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1244, new segment cells count=444 2024-12-02T21:15:54,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=444, cellSize=20334, totalHeapSize=46416, min timestamp=1733174154368, max timestamp=1733174154382Num uniques 1244; 2024-12-02T21:15:54,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=56974, new segment data size=20334, suffix heap size=157344,new segment heap size=46416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1244, new segment cells count=444 2024-12-02T21:15:54,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=444, cellSize=20334, totalHeapSize=46416, min timestamp=1733174154368, max timestamp=1733174154382Num uniques 1094; 2024-12-02T21:15:54,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=20334, suffix heap size=141744,new segment heap size=46416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=444 2024-12-02T21:15:54,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174214400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174214401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174214402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174214404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174214405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174214510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174214510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174214510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174214510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174214518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,537 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/0c5005349f9b4580bcb55aff43739704 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0c5005349f9b4580bcb55aff43739704 2024-12-02T21:15:54,545 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/A of ab928e240d6dc631659396e2f2d0cdda into 0c5005349f9b4580bcb55aff43739704(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:54,545 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:54,545 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/A, priority=13, startTime=1733174153575; duration=0sec 2024-12-02T21:15:54,546 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:54,546 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:A 2024-12-02T21:15:54,559 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/23973d9992454b7d87a1b800aac92f76 2024-12-02T21:15:54,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/ef1f099bf4804a29a244d3ac73c34b6b is 50, key is test_row_0/C:col10/1733174152593/Put/seqid=0 2024-12-02T21:15:54,584 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/78aa4cdb8b3b41a3b149b56edb83e2c5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/78aa4cdb8b3b41a3b149b56edb83e2c5 2024-12-02T21:15:54,590 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/B of ab928e240d6dc631659396e2f2d0cdda into 78aa4cdb8b3b41a3b149b56edb83e2c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:54,590 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:54,591 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/B, priority=13, startTime=1733174153575; duration=0sec 2024-12-02T21:15:54,591 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:54,591 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:B 2024-12-02T21:15:54,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742000_1176 (size=9857) 2024-12-02T21:15:54,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174214719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174214720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174214720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174214720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174214728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T21:15:55,004 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/ef1f099bf4804a29a244d3ac73c34b6b 2024-12-02T21:15:55,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8d1cdef2dcb34f03bc10f26d4204aaac as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8d1cdef2dcb34f03bc10f26d4204aaac 2024-12-02T21:15:55,014 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8d1cdef2dcb34f03bc10f26d4204aaac, entries=100, sequenceid=355, filesize=22.0 K 2024-12-02T21:15:55,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/23973d9992454b7d87a1b800aac92f76 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/23973d9992454b7d87a1b800aac92f76 2024-12-02T21:15:55,024 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/23973d9992454b7d87a1b800aac92f76, entries=100, sequenceid=355, filesize=9.6 K 2024-12-02T21:15:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/ef1f099bf4804a29a244d3ac73c34b6b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ef1f099bf4804a29a244d3ac73c34b6b 2024-12-02T21:15:55,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174215029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174215029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174215029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174215030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,037 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ef1f099bf4804a29a244d3ac73c34b6b, entries=100, sequenceid=355, filesize=9.6 K 2024-12-02T21:15:55,038 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=174.43 KB/178620 for ab928e240d6dc631659396e2f2d0cdda in 1459ms, sequenceid=355, compaction requested=false 2024-12-02T21:15:55,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:55,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:55,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-02T21:15:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-02T21:15:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:55,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=181.14 KB heapSize=498.42 KB 2024-12-02T21:15:55,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:55,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T21:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T21:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T21:15:55,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-02T21:15:55,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2270 sec 2024-12-02T21:15:55,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2360 sec 2024-12-02T21:15:55,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174215050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020150afe1ef014345b32668c2fe47cddd_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174154404/Put/seqid=0 2024-12-02T21:15:55,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742001_1177 (size=12454) 2024-12-02T21:15:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174215159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174215368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:55,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020150afe1ef014345b32668c2fe47cddd_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020150afe1ef014345b32668c2fe47cddd_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:55,467 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/3c3b114fe4484fd6b66f8040b525f661, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:55,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/3c3b114fe4484fd6b66f8040b525f661 is 175, key is test_row_0/A:col10/1733174154404/Put/seqid=0 2024-12-02T21:15:55,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742002_1178 (size=31255) 2024-12-02T21:15:55,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174215539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174215539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174215539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174215539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174215679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:55,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=654, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/3c3b114fe4484fd6b66f8040b525f661 2024-12-02T21:15:55,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/a813e7f08ae3465e8437bac0c65830af is 50, key is test_row_0/B:col10/1733174154404/Put/seqid=0 2024-12-02T21:15:55,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742003_1179 (size=12301) 2024-12-02T21:15:56,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174216188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:56,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=654 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/a813e7f08ae3465e8437bac0c65830af 2024-12-02T21:15:56,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/7c9a5e74f5fb4c2ab7f861f5f8fff530 is 50, key is test_row_0/C:col10/1733174154404/Put/seqid=0 2024-12-02T21:15:56,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742004_1180 (size=12301) 2024-12-02T21:15:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174216549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174216549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:56,550 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:56,551 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:56,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174216559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:56,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174216559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:56,562 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:56,563 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:56,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=654 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/7c9a5e74f5fb4c2ab7f861f5f8fff530 2024-12-02T21:15:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/3c3b114fe4484fd6b66f8040b525f661 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3c3b114fe4484fd6b66f8040b525f661 2024-12-02T21:15:56,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3c3b114fe4484fd6b66f8040b525f661, entries=150, sequenceid=654, filesize=30.5 K 2024-12-02T21:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/a813e7f08ae3465e8437bac0c65830af as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/a813e7f08ae3465e8437bac0c65830af 2024-12-02T21:15:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/a813e7f08ae3465e8437bac0c65830af, entries=150, sequenceid=654, filesize=12.0 K 2024-12-02T21:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/7c9a5e74f5fb4c2ab7f861f5f8fff530 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/7c9a5e74f5fb4c2ab7f861f5f8fff530 2024-12-02T21:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/7c9a5e74f5fb4c2ab7f861f5f8fff530, entries=150, sequenceid=654, filesize=12.0 K 2024-12-02T21:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~498.38 KB/510336, currentSize=6.71 KB/6870 for ab928e240d6dc631659396e2f2d0cdda in 1707ms, sequenceid=654, compaction requested=true 2024-12-02T21:15:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,749 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:56,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:56,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:56,749 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:56,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:56,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,750 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,750 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/B is initiating minor compaction (all files) 2024-12-02T21:15:56,751 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/B in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,751 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/78aa4cdb8b3b41a3b149b56edb83e2c5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/23973d9992454b7d87a1b800aac92f76, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/a813e7f08ae3465e8437bac0c65830af] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=33.9 K 2024-12-02T21:15:56,751 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:56,751 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/A is initiating minor compaction (all files) 2024-12-02T21:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,751 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/A in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:56,751 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0c5005349f9b4580bcb55aff43739704, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8d1cdef2dcb34f03bc10f26d4204aaac, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3c3b114fe4484fd6b66f8040b525f661] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=83.3 K 2024-12-02T21:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,751 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:56,751 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0c5005349f9b4580bcb55aff43739704, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8d1cdef2dcb34f03bc10f26d4204aaac, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3c3b114fe4484fd6b66f8040b525f661] 2024-12-02T21:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,752 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 78aa4cdb8b3b41a3b149b56edb83e2c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733174152251 2024-12-02T21:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,752 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 23973d9992454b7d87a1b800aac92f76, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733174152593 2024-12-02T21:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,753 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c5005349f9b4580bcb55aff43739704, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733174152251 2024-12-02T21:15:56,753 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting a813e7f08ae3465e8437bac0c65830af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1733174154401 2024-12-02T21:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,754 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d1cdef2dcb34f03bc10f26d4204aaac, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733174152593 2024-12-02T21:15:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,754 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c3b114fe4484fd6b66f8040b525f661, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1733174154401 2024-12-02T21:15:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,765 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,766 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,767 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/bd7a46e0348c41acba65e071e0cca8ea is 50, key is test_row_0/B:col10/1733174154404/Put/seqid=0 2024-12-02T21:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,777 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412020c991cc82f16428cb86778445d6f56eb_ab928e240d6dc631659396e2f2d0cdda store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,778 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412020c991cc82f16428cb86778445d6f56eb_ab928e240d6dc631659396e2f2d0cdda, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:56,778 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020c991cc82f16428cb86778445d6f56eb_ab928e240d6dc631659396e2f2d0cdda because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742005_1181 (size=12643) 2024-12-02T21:15:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,820 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/bd7a46e0348c41acba65e071e0cca8ea as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bd7a46e0348c41acba65e071e0cca8ea 2024-12-02T21:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,826 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/B of ab928e240d6dc631659396e2f2d0cdda into bd7a46e0348c41acba65e071e0cca8ea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:56,826 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:56,826 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/B, priority=13, startTime=1733174156749; duration=0sec 2024-12-02T21:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,826 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:56,826 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:B 2024-12-02T21:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,826 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,828 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:56,828 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/C is initiating minor compaction (all files) 2024-12-02T21:15:56,828 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/C in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:56,828 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/123dd432968645f58d675bc258ad0b0e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ef1f099bf4804a29a244d3ac73c34b6b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/7c9a5e74f5fb4c2ab7f861f5f8fff530] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=33.9 K 2024-12-02T21:15:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,829 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 123dd432968645f58d675bc258ad0b0e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733174152251 2024-12-02T21:15:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,830 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ef1f099bf4804a29a244d3ac73c34b6b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733174152593 2024-12-02T21:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,830 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c9a5e74f5fb4c2ab7f861f5f8fff530, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1733174154401 2024-12-02T21:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742006_1182 (size=4469) 2024-12-02T21:15:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,858 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#C#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,858 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/af1f6805911b490a8c062f0ed4a0fc36 is 50, key is test_row_0/C:col10/1733174154404/Put/seqid=0 2024-12-02T21:15:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742007_1183 (size=12643) 2024-12-02T21:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,903 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/af1f6805911b490a8c062f0ed4a0fc36 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/af1f6805911b490a8c062f0ed4a0fc36 2024-12-02T21:15:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,910 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/C of ab928e240d6dc631659396e2f2d0cdda into af1f6805911b490a8c062f0ed4a0fc36(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:56,910 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:56,910 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/C, priority=13, startTime=1733174156749; duration=0sec 2024-12-02T21:15:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,910 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:56,911 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:C 2024-12-02T21:15:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T21:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,980 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,980 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-02T21:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,983 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T21:15:56,985 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T21:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T21:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:57,138 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-02T21:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ca3123bb8a864a3d9e4a47ec2c06d7a0_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_2/A:col10/1733174155048/Put/seqid=0 2024-12-02T21:15:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742008_1184 (size=7374) 2024-12-02T21:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,231 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ca3123bb8a864a3d9e4a47ec2c06d7a0_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ca3123bb8a864a3d9e4a47ec2c06d7a0_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8b88fdc493d1433d927c4bcb933aa33e, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8b88fdc493d1433d927c4bcb933aa33e is 175, key is test_row_2/A:col10/1733174155048/Put/seqid=0 2024-12-02T21:15:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742009_1185 (size=13865) 2024-12-02T21:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,253 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=660, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8b88fdc493d1433d927c4bcb933aa33e 2024-12-02T21:15:57,253 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#A#compaction#155 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,254 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/d8615daf63ab4b81800926a36e627a6e is 175, key is test_row_0/A:col10/1733174154404/Put/seqid=0 2024-12-02T21:15:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/3242ab0857c044d5b4b6ab220a9e4dab is 50, key is test_row_2/B:col10/1733174155048/Put/seqid=0 2024-12-02T21:15:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742010_1186 (size=31597) 2024-12-02T21:15:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T21:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742011_1187 (size=7415) 2024-12-02T21:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,316 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=660 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/3242ab0857c044d5b4b6ab220a9e4dab 2024-12-02T21:15:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/455d1256178f41fa80397c43da810fd9 is 50, key is test_row_2/C:col10/1733174155048/Put/seqid=0 2024-12-02T21:15:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:57,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742012_1188 (size=7415) 2024-12-02T21:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174157397, max timestamp=1733174157413Num uniques 859; 2024-12-02T21:15:57,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:57,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174157397, max timestamp=1733174157413Num uniques 859; 2024-12-02T21:15:57,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:57,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174157397, max timestamp=1733174157413Num uniques 859; 2024-12-02T21:15:57,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:57,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174157467, max timestamp=1733174157502Num uniques 1018; 2024-12-02T21:15:57,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-02T21:15:57,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174157467, max timestamp=1733174157502Num uniques 1018; 2024-12-02T21:15:57,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-02T21:15:57,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174157467, max timestamp=1733174157502Num uniques 1018; 2024-12-02T21:15:57,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-02T21:15:57,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733174157572, max timestamp=1733174157597Num uniques 1077; 2024-12-02T21:15:57,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-02T21:15:57,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733174157572, max timestamp=1733174157597Num uniques 1077; 2024-12-02T21:15:57,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-02T21:15:57,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733174157572, max timestamp=1733174157597Num uniques 1077; 2024-12-02T21:15:57,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-02T21:15:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T21:15:57,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733174157639, max timestamp=1733174157665Num uniques 1086; 2024-12-02T21:15:57,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-02T21:15:57,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733174157639, max timestamp=1733174157665Num uniques 1086; 2024-12-02T21:15:57,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733174157639, max timestamp=1733174157665Num uniques 1036; 2024-12-02T21:15:57,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-02T21:15:57,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-02T21:15:57,694 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/d8615daf63ab4b81800926a36e627a6e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/d8615daf63ab4b81800926a36e627a6e 2024-12-02T21:15:57,732 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/A of ab928e240d6dc631659396e2f2d0cdda into d8615daf63ab4b81800926a36e627a6e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:57,732 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:57,732 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/A, priority=13, startTime=1733174156748; duration=0sec 2024-12-02T21:15:57,733 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:57,733 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:A 2024-12-02T21:15:57,751 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=660 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/455d1256178f41fa80397c43da810fd9 2024-12-02T21:15:57,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733174157684, max timestamp=1733174157758Num uniques 1095; 2024-12-02T21:15:57,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-02T21:15:57,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733174157684, max timestamp=1733174157758Num uniques 1095; 2024-12-02T21:15:57,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-02T21:15:57,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733174157684, max timestamp=1733174157758Num uniques 1095; 2024-12-02T21:15:57,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-02T21:15:57,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8b88fdc493d1433d927c4bcb933aa33e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8b88fdc493d1433d927c4bcb933aa33e 2024-12-02T21:15:57,783 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8b88fdc493d1433d927c4bcb933aa33e, entries=50, sequenceid=660, filesize=13.5 K 2024-12-02T21:15:57,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/3242ab0857c044d5b4b6ab220a9e4dab as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/3242ab0857c044d5b4b6ab220a9e4dab 2024-12-02T21:15:57,796 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/3242ab0857c044d5b4b6ab220a9e4dab, entries=50, sequenceid=660, filesize=7.2 K 2024-12-02T21:15:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/455d1256178f41fa80397c43da810fd9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/455d1256178f41fa80397c43da810fd9 2024-12-02T21:15:57,808 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/455d1256178f41fa80397c43da810fd9, entries=50, sequenceid=660, filesize=7.2 K 2024-12-02T21:15:57,815 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=51.44 KB/52670 for ab928e240d6dc631659396e2f2d0cdda in 677ms, sequenceid=660, compaction requested=false 2024-12-02T21:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-02T21:15:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:57,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=58.14 KB heapSize=152.70 KB 2024-12-02T21:15:57,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:57,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:57,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:57,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:57,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:57,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-02T21:15:57,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-02T21:15:57,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 833 msec 2024-12-02T21:15:57,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 841 msec 2024-12-02T21:15:57,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021a83804a5abd4eac9a8929bb29d427c2_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174157779/Put/seqid=0 2024-12-02T21:15:57,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742013_1189 (size=14994) 2024-12-02T21:15:57,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:57,888 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021a83804a5abd4eac9a8929bb29d427c2_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021a83804a5abd4eac9a8929bb29d427c2_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:57,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/04acd926ff8540ad90248b78e17bbfa2, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:57,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/04acd926ff8540ad90248b78e17bbfa2 is 175, key is test_row_0/A:col10/1733174157779/Put/seqid=0 2024-12-02T21:15:57,894 INFO [master/87c3fdb6c570:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T21:15:57,894 INFO [master/87c3fdb6c570:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T21:15:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742014_1190 (size=39949) 2024-12-02T21:15:57,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=754, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/04acd926ff8540ad90248b78e17bbfa2 2024-12-02T21:15:57,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/bc1b9188dbc14f7b9d6a5dea86e608b7 is 50, key is test_row_0/B:col10/1733174157779/Put/seqid=0 2024-12-02T21:15:57,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174157884, max timestamp=1733174157951Num uniques 859; 2024-12-02T21:15:57,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174157884, max timestamp=1733174157951Num uniques 859; 2024-12-02T21:15:57,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:57,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:57,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174157884, max timestamp=1733174157951Num uniques 859; 2024-12-02T21:15:57,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:15:57,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742015_1191 (size=12301) 2024-12-02T21:15:58,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174218102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T21:15:58,119 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:15:58,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:15:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-02T21:15:58,121 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:15:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T21:15:58,122 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:15:58,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:15:58,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174218218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T21:15:58,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/bc1b9188dbc14f7b9d6a5dea86e608b7 2024-12-02T21:15:58,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/68548fc8d4974e9c83ea42bde3178be4 is 50, key is test_row_0/C:col10/1733174157779/Put/seqid=0 2024-12-02T21:15:58,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174218429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T21:15:58,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742016_1192 (size=12301) 2024-12-02T21:15:58,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/68548fc8d4974e9c83ea42bde3178be4 2024-12-02T21:15:58,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/04acd926ff8540ad90248b78e17bbfa2 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/04acd926ff8540ad90248b78e17bbfa2 2024-12-02T21:15:58,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/04acd926ff8540ad90248b78e17bbfa2, entries=200, sequenceid=754, filesize=39.0 K 2024-12-02T21:15:58,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/bc1b9188dbc14f7b9d6a5dea86e608b7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bc1b9188dbc14f7b9d6a5dea86e608b7 2024-12-02T21:15:58,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bc1b9188dbc14f7b9d6a5dea86e608b7, entries=150, sequenceid=754, filesize=12.0 K 2024-12-02T21:15:58,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/68548fc8d4974e9c83ea42bde3178be4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/68548fc8d4974e9c83ea42bde3178be4 2024-12-02T21:15:58,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/68548fc8d4974e9c83ea42bde3178be4, entries=150, sequenceid=754, filesize=12.0 K 2024-12-02T21:15:58,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~152.66 KB/156320, currentSize=127.47 KB/130530 for ab928e240d6dc631659396e2f2d0cdda in 659ms, sequenceid=754, compaction requested=true 2024-12-02T21:15:58,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:58,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:15:58,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:58,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:15:58,476 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:58,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:58,476 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:58,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:15:58,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:58,477 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:58,477 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/B is initiating minor compaction (all files) 2024-12-02T21:15:58,477 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85411 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:58,478 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/B in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,478 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/A is initiating minor compaction (all files) 2024-12-02T21:15:58,478 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/A in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,478 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bd7a46e0348c41acba65e071e0cca8ea, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/3242ab0857c044d5b4b6ab220a9e4dab, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bc1b9188dbc14f7b9d6a5dea86e608b7] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=31.6 K 2024-12-02T21:15:58,478 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/d8615daf63ab4b81800926a36e627a6e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8b88fdc493d1433d927c4bcb933aa33e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/04acd926ff8540ad90248b78e17bbfa2] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=83.4 K 2024-12-02T21:15:58,478 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,478 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/d8615daf63ab4b81800926a36e627a6e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8b88fdc493d1433d927c4bcb933aa33e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/04acd926ff8540ad90248b78e17bbfa2] 2024-12-02T21:15:58,478 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting bd7a46e0348c41acba65e071e0cca8ea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1733174154401 2024-12-02T21:15:58,478 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8615daf63ab4b81800926a36e627a6e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1733174154401 2024-12-02T21:15:58,478 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 3242ab0857c044d5b4b6ab220a9e4dab, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1733174155048 2024-12-02T21:15:58,479 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b88fdc493d1433d927c4bcb933aa33e, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1733174155048 2024-12-02T21:15:58,479 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting bc1b9188dbc14f7b9d6a5dea86e608b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=754, earliestPutTs=1733174157779 2024-12-02T21:15:58,479 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04acd926ff8540ad90248b78e17bbfa2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=754, earliestPutTs=1733174157758 2024-12-02T21:15:58,505 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#B#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:58,505 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/9ae51c04f09545c299868f72ad6dc617 is 50, key is test_row_0/B:col10/1733174157779/Put/seqid=0 2024-12-02T21:15:58,509 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:58,570 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202f0d64c5daee64e148b216c9e84839cad_ab928e240d6dc631659396e2f2d0cdda store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:58,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T21:15:58,572 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202f0d64c5daee64e148b216c9e84839cad_ab928e240d6dc631659396e2f2d0cdda, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:58,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:15:58,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:58,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:15:58,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:58,572 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f0d64c5daee64e148b216c9e84839cad_ab928e240d6dc631659396e2f2d0cdda because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:58,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:15:58,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:15:58,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742017_1193 (size=12745) 2024-12-02T21:15:58,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174218588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742018_1194 (size=4469) 2024-12-02T21:15:58,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174218589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174218591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174218591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,595 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026b62b5b538c542a3bbc291a22d4b76f6_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174158095/Put/seqid=0 2024-12-02T21:15:58,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742019_1195 (size=14994) 2024-12-02T21:15:58,655 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:15:58,660 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026b62b5b538c542a3bbc291a22d4b76f6_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026b62b5b538c542a3bbc291a22d4b76f6_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:15:58,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/1a20a44d63a94d5a8193546c02870e5a, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:15:58,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/1a20a44d63a94d5a8193546c02870e5a is 175, key is test_row_0/A:col10/1733174158095/Put/seqid=0 2024-12-02T21:15:58,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742020_1196 (size=39949) 2024-12-02T21:15:58,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174218698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174218699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174218699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174218739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:58,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:58,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T21:15:58,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:58,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:58,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:58,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:58,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174218909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174218909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174218910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:58,985 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/9ae51c04f09545c299868f72ad6dc617 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/9ae51c04f09545c299868f72ad6dc617 2024-12-02T21:15:58,994 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/B of ab928e240d6dc631659396e2f2d0cdda into 9ae51c04f09545c299868f72ad6dc617(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:58,994 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:58,994 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/B, priority=13, startTime=1733174158476; duration=0sec 2024-12-02T21:15:58,995 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:15:58,995 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:B 2024-12-02T21:15:58,995 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:15:58,995 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#A#compaction#165 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:58,996 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8a7301d4f40e48d5aa9ce74853aa5ae6 is 175, key is test_row_0/A:col10/1733174157779/Put/seqid=0 2024-12-02T21:15:59,000 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:15:59,000 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/C is initiating minor compaction (all files) 2024-12-02T21:15:59,000 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/C in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,000 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/af1f6805911b490a8c062f0ed4a0fc36, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/455d1256178f41fa80397c43da810fd9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/68548fc8d4974e9c83ea42bde3178be4] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=31.6 K 2024-12-02T21:15:59,001 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting af1f6805911b490a8c062f0ed4a0fc36, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1733174154401 2024-12-02T21:15:59,002 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 455d1256178f41fa80397c43da810fd9, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1733174155048 2024-12-02T21:15:59,002 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 68548fc8d4974e9c83ea42bde3178be4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=754, earliestPutTs=1733174157779 2024-12-02T21:15:59,028 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:15:59,029 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/e6e6da7058b94284890d5211a7d6f906 is 50, key is test_row_0/C:col10/1733174157779/Put/seqid=0 2024-12-02T21:15:59,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742021_1197 (size=31699) 2024-12-02T21:15:59,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742022_1198 (size=12745) 2024-12-02T21:15:59,078 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/e6e6da7058b94284890d5211a7d6f906 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/e6e6da7058b94284890d5211a7d6f906 2024-12-02T21:15:59,087 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/C of ab928e240d6dc631659396e2f2d0cdda into e6e6da7058b94284890d5211a7d6f906(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:59,087 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:59,087 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/C, priority=13, startTime=1733174158476; duration=0sec 2024-12-02T21:15:59,088 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:59,088 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:C 2024-12-02T21:15:59,099 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=791, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/1a20a44d63a94d5a8193546c02870e5a 2024-12-02T21:15:59,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/6baf37b889a84023a6eae588111b381e is 50, key is test_row_0/B:col10/1733174158095/Put/seqid=0 2024-12-02T21:15:59,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742023_1199 (size=12301) 2024-12-02T21:15:59,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174219219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:59,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174219219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174219228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:59,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174219249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T21:15:59,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,435 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/8a7301d4f40e48d5aa9ce74853aa5ae6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8a7301d4f40e48d5aa9ce74853aa5ae6 2024-12-02T21:15:59,455 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/A of ab928e240d6dc631659396e2f2d0cdda into 8a7301d4f40e48d5aa9ce74853aa5ae6(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:15:59,455 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:15:59,455 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/A, priority=13, startTime=1733174158476; duration=0sec 2024-12-02T21:15:59,455 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:15:59,455 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:A 2024-12-02T21:15:59,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=791 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/6baf37b889a84023a6eae588111b381e 2024-12-02T21:15:59,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/8c481ce10f62411a91a202724a47ee1c is 50, key is test_row_0/C:col10/1733174158095/Put/seqid=0 2024-12-02T21:15:59,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742024_1200 (size=12301) 2024-12-02T21:15:59,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:59,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:59,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174219728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:59,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174219729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:15:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174219759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:15:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:15:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:15:59,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:59,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:59,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:15:59,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:15:59,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:15:59,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:15:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:00,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=791 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/8c481ce10f62411a91a202724a47ee1c 2024-12-02T21:16:00,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/1a20a44d63a94d5a8193546c02870e5a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1a20a44d63a94d5a8193546c02870e5a 2024-12-02T21:16:00,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1a20a44d63a94d5a8193546c02870e5a, entries=200, sequenceid=791, filesize=39.0 K 2024-12-02T21:16:00,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/6baf37b889a84023a6eae588111b381e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6baf37b889a84023a6eae588111b381e 2024-12-02T21:16:00,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6baf37b889a84023a6eae588111b381e, entries=150, sequenceid=791, filesize=12.0 K 2024-12-02T21:16:00,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/8c481ce10f62411a91a202724a47ee1c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/8c481ce10f62411a91a202724a47ee1c 2024-12-02T21:16:00,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/8c481ce10f62411a91a202724a47ee1c, entries=150, sequenceid=791, filesize=12.0 K 2024-12-02T21:16:00,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ab928e240d6dc631659396e2f2d0cdda in 1485ms, sequenceid=791, compaction requested=false 2024-12-02T21:16:00,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:00,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T21:16:00,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:00,149 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:16:00,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:16:00,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:00,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:16:00,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:00,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:16:00,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:00,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e1b0c1ced43641eb945d8263b16f4623_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174158583/Put/seqid=0 2024-12-02T21:16:00,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742025_1201 (size=12454) 2024-12-02T21:16:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T21:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:00,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:00,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174160304, max timestamp=1733174160317Num uniques 859; 2024-12-02T21:16:00,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:00,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174160304, max timestamp=1733174160317Num uniques 859; 2024-12-02T21:16:00,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:00,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174160304, max timestamp=1733174160317Num uniques 859; 2024-12-02T21:16:00,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:00,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 356 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174220393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:00,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:00,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 357 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174220499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:00,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:00,576 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e1b0c1ced43641eb945d8263b16f4623_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e1b0c1ced43641eb945d8263b16f4623_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/39e7903fc4f94298bbf86467d2dba02c, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:00,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/39e7903fc4f94298bbf86467d2dba02c is 175, key is test_row_0/A:col10/1733174158583/Put/seqid=0 2024-12-02T21:16:00,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 358 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174220710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:00,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:00,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174220739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:00,746 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:00,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174220748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:00,758 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:00,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174220769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:00,773 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:00,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742026_1202 (size=31255) 2024-12-02T21:16:01,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 359 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174221019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:01,237 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=804, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/39e7903fc4f94298bbf86467d2dba02c 2024-12-02T21:16:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/08ef853cd8d84f57a5e9794e37783b18 is 50, key is test_row_0/B:col10/1733174158583/Put/seqid=0 2024-12-02T21:16:01,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742027_1203 (size=12301) 2024-12-02T21:16:01,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 360 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174221529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:01,657 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/08ef853cd8d84f57a5e9794e37783b18 2024-12-02T21:16:01,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/029499031b7c4b94899332f235c0dca3 is 50, key is test_row_0/C:col10/1733174158583/Put/seqid=0 2024-12-02T21:16:01,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742028_1204 (size=12301) 2024-12-02T21:16:02,074 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/029499031b7c4b94899332f235c0dca3 2024-12-02T21:16:02,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/39e7903fc4f94298bbf86467d2dba02c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39e7903fc4f94298bbf86467d2dba02c 2024-12-02T21:16:02,082 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39e7903fc4f94298bbf86467d2dba02c, entries=150, sequenceid=804, filesize=30.5 K 2024-12-02T21:16:02,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/08ef853cd8d84f57a5e9794e37783b18 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/08ef853cd8d84f57a5e9794e37783b18 2024-12-02T21:16:02,087 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/08ef853cd8d84f57a5e9794e37783b18, entries=150, sequenceid=804, filesize=12.0 K 2024-12-02T21:16:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/029499031b7c4b94899332f235c0dca3 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/029499031b7c4b94899332f235c0dca3 2024-12-02T21:16:02,091 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/029499031b7c4b94899332f235c0dca3, entries=150, sequenceid=804, filesize=12.0 K 2024-12-02T21:16:02,092 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ab928e240d6dc631659396e2f2d0cdda in 1943ms, sequenceid=804, compaction requested=true 2024-12-02T21:16:02,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-02T21:16:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-02T21:16:02,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-02T21:16:02,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9730 sec 2024-12-02T21:16:02,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.9790 sec 2024-12-02T21:16:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T21:16:02,299 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:02,299 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-02T21:16:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T21:16:02,301 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:02,302 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:02,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T21:16:02,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T21:16:02,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:02,454 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T21:16:02,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:16:02,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:02,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:16:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:16:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d10e7baf674b4e90a0ad5cd25602459b_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174160391/Put/seqid=0 2024-12-02T21:16:02,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742029_1205 (size=12454) 2024-12-02T21:16:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:02,471 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d10e7baf674b4e90a0ad5cd25602459b_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d10e7baf674b4e90a0ad5cd25602459b_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/f7b9b90b1c1743069fe565587277c5b1, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/f7b9b90b1c1743069fe565587277c5b1 is 175, key is test_row_0/A:col10/1733174160391/Put/seqid=0 2024-12-02T21:16:02,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742030_1206 (size=31255) 2024-12-02T21:16:02,483 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=841, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/f7b9b90b1c1743069fe565587277c5b1 2024-12-02T21:16:02,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/b6e8a8b82c6640329f58fb01730e9063 is 50, key is test_row_0/B:col10/1733174160391/Put/seqid=0 2024-12-02T21:16:02,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742031_1207 (size=12301) 2024-12-02T21:16:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:02,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:02,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 368 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174222581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T21:16:02,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58218 deadline: 1733174222639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:02,642 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:02,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 369 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174222688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:02,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:02,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174222769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:02,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174222769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:02,773 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:02,773 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:02,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174222788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:02,792 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:02,897 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=841 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/b6e8a8b82c6640329f58fb01730e9063 2024-12-02T21:16:02,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 370 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174222898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:02,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/f8981f9fa30f400988f74e541d51b304 is 50, key is test_row_0/C:col10/1733174160391/Put/seqid=0 2024-12-02T21:16:02,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742032_1208 (size=12301) 2024-12-02T21:16:02,915 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=841 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/f8981f9fa30f400988f74e541d51b304 2024-12-02T21:16:02,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/f7b9b90b1c1743069fe565587277c5b1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/f7b9b90b1c1743069fe565587277c5b1 2024-12-02T21:16:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T21:16:02,934 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/f7b9b90b1c1743069fe565587277c5b1, entries=150, sequenceid=841, filesize=30.5 K 2024-12-02T21:16:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/b6e8a8b82c6640329f58fb01730e9063 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b6e8a8b82c6640329f58fb01730e9063 2024-12-02T21:16:02,941 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b6e8a8b82c6640329f58fb01730e9063, entries=150, sequenceid=841, filesize=12.0 K 2024-12-02T21:16:02,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/f8981f9fa30f400988f74e541d51b304 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f8981f9fa30f400988f74e541d51b304 2024-12-02T21:16:02,954 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f8981f9fa30f400988f74e541d51b304, entries=150, sequenceid=841, filesize=12.0 K 2024-12-02T21:16:02,955 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ab928e240d6dc631659396e2f2d0cdda in 501ms, sequenceid=841, compaction requested=true 2024-12-02T21:16:02,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:02,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:02,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-02T21:16:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-02T21:16:02,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-02T21:16:02,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 656 msec 2024-12-02T21:16:02,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 663 msec 2024-12-02T21:16:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:03,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:16:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:16:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:16:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:16:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:03,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202439550b897264deb80b69314a9375119_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174162568/Put/seqid=0 2024-12-02T21:16:03,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742033_1209 (size=12454) 2024-12-02T21:16:03,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:03,226 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202439550b897264deb80b69314a9375119_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202439550b897264deb80b69314a9375119_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:03,227 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/014df2799dbd467c9efc0b071bc77039, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:03,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/014df2799dbd467c9efc0b071bc77039 is 175, key is test_row_0/A:col10/1733174162568/Put/seqid=0 2024-12-02T21:16:03,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742034_1210 (size=31255) 2024-12-02T21:16:03,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174163262, max timestamp=1733174163286Num uniques 859; 2024-12-02T21:16:03,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174163262, max timestamp=1733174163286Num uniques 859; 2024-12-02T21:16:03,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:16:03,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:16:03,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174163262, max timestamp=1733174163286Num uniques 859; 2024-12-02T21:16:03,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:16:03,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 403 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174223353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T21:16:03,439 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:03,440 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-02T21:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T21:16:03,441 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:03,442 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:03,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:03,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 404 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174223458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T21:16:03,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T21:16:03,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:03,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:03,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:03,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:03,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:03,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=852, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/014df2799dbd467c9efc0b071bc77039 2024-12-02T21:16:03,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/da1c950936e345cbb3cd659041b991c3 is 50, key is test_row_0/B:col10/1733174162568/Put/seqid=0 2024-12-02T21:16:03,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742035_1211 (size=12301) 2024-12-02T21:16:03,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=852 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/da1c950936e345cbb3cd659041b991c3 2024-12-02T21:16:03,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/d7a4cef320ce47dd96950bb246be9717 is 50, key is test_row_0/C:col10/1733174162568/Put/seqid=0 2024-12-02T21:16:03,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742036_1212 (size=12301) 2024-12-02T21:16:03,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 405 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174223669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:03,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T21:16:03,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:03,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:03,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:03,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:03,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T21:16:03,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T21:16:03,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:03,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:03,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:03,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:03,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:03,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 406 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174223979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:04,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T21:16:04,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:04,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:04,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:04,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:04,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:04,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=852 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/d7a4cef320ce47dd96950bb246be9717 2024-12-02T21:16:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T21:16:04,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/014df2799dbd467c9efc0b071bc77039 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/014df2799dbd467c9efc0b071bc77039 2024-12-02T21:16:04,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/014df2799dbd467c9efc0b071bc77039, entries=150, sequenceid=852, filesize=30.5 K 2024-12-02T21:16:04,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/da1c950936e345cbb3cd659041b991c3 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/da1c950936e345cbb3cd659041b991c3 2024-12-02T21:16:04,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/da1c950936e345cbb3cd659041b991c3, entries=150, sequenceid=852, filesize=12.0 K 2024-12-02T21:16:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/d7a4cef320ce47dd96950bb246be9717 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/d7a4cef320ce47dd96950bb246be9717 2024-12-02T21:16:04,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/d7a4cef320ce47dd96950bb246be9717, entries=150, sequenceid=852, filesize=12.0 K 2024-12-02T21:16:04,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ab928e240d6dc631659396e2f2d0cdda in 878ms, sequenceid=852, compaction requested=true 2024-12-02T21:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:04,088 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T21:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T21:16:04,090 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T21:16:04,090 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165413 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T21:16:04,090 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/A is initiating minor compaction (all files) 2024-12-02T21:16:04,090 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/A in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:04,090 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8a7301d4f40e48d5aa9ce74853aa5ae6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1a20a44d63a94d5a8193546c02870e5a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39e7903fc4f94298bbf86467d2dba02c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/f7b9b90b1c1743069fe565587277c5b1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/014df2799dbd467c9efc0b071bc77039] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=161.5 K 2024-12-02T21:16:04,090 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:04,090 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8a7301d4f40e48d5aa9ce74853aa5ae6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1a20a44d63a94d5a8193546c02870e5a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39e7903fc4f94298bbf86467d2dba02c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/f7b9b90b1c1743069fe565587277c5b1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/014df2799dbd467c9efc0b071bc77039] 2024-12-02T21:16:04,091 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7301d4f40e48d5aa9ce74853aa5ae6, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=754, earliestPutTs=1733174157779 2024-12-02T21:16:04,091 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a20a44d63a94d5a8193546c02870e5a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=791, earliestPutTs=1733174158078 2024-12-02T21:16:04,091 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61949 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T21:16:04,091 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/B is initiating minor compaction (all files) 2024-12-02T21:16:04,091 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/B in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:04,091 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/9ae51c04f09545c299868f72ad6dc617, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6baf37b889a84023a6eae588111b381e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/08ef853cd8d84f57a5e9794e37783b18, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b6e8a8b82c6640329f58fb01730e9063, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/da1c950936e345cbb3cd659041b991c3] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=60.5 K 2024-12-02T21:16:04,092 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39e7903fc4f94298bbf86467d2dba02c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=804, earliestPutTs=1733174158575 2024-12-02T21:16:04,092 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae51c04f09545c299868f72ad6dc617, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=754, earliestPutTs=1733174157779 2024-12-02T21:16:04,092 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b9b90b1c1743069fe565587277c5b1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=841, earliestPutTs=1733174160369 2024-12-02T21:16:04,092 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6baf37b889a84023a6eae588111b381e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=791, earliestPutTs=1733174158078 2024-12-02T21:16:04,092 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ef853cd8d84f57a5e9794e37783b18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=804, earliestPutTs=1733174158575 2024-12-02T21:16:04,092 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 014df2799dbd467c9efc0b071bc77039, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=852, earliestPutTs=1733174162568 2024-12-02T21:16:04,093 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e8a8b82c6640329f58fb01730e9063, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=841, earliestPutTs=1733174160369 2024-12-02T21:16:04,093 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting da1c950936e345cbb3cd659041b991c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=852, earliestPutTs=1733174162568 2024-12-02T21:16:04,101 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:04,104 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#B#compaction#180 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:04,104 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/6bc6567fd6bd4deb8014cfeb2d3a0e02 is 50, key is test_row_0/B:col10/1733174162568/Put/seqid=0 2024-12-02T21:16:04,107 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202712eeedaef9c49dcb20864270e948548_ab928e240d6dc631659396e2f2d0cdda store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:04,110 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202712eeedaef9c49dcb20864270e948548_ab928e240d6dc631659396e2f2d0cdda, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:04,110 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202712eeedaef9c49dcb20864270e948548_ab928e240d6dc631659396e2f2d0cdda because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:04,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742037_1213 (size=12915) 2024-12-02T21:16:04,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742038_1214 (size=4469) 2024-12-02T21:16:04,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T21:16:04,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:04,211 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-02T21:16:04,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:16:04,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:16:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:16:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:04,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025d6c8827188042d7b5d512a9f4b184aa_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174163352/Put/seqid=0 2024-12-02T21:16:04,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742039_1215 (size=12454) 2024-12-02T21:16:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:04,518 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/6bc6567fd6bd4deb8014cfeb2d3a0e02 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6bc6567fd6bd4deb8014cfeb2d3a0e02 2024-12-02T21:16:04,518 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#A#compaction#179 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:04,519 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/3018caf4cb204e0f82892f9079537176 is 175, key is test_row_0/A:col10/1733174162568/Put/seqid=0 2024-12-02T21:16:04,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742040_1216 (size=31869) 2024-12-02T21:16:04,523 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/B of ab928e240d6dc631659396e2f2d0cdda into 6bc6567fd6bd4deb8014cfeb2d3a0e02(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:04,523 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:04,523 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/B, priority=11, startTime=1733174164088; duration=0sec 2024-12-02T21:16:04,523 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:04,523 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:B 2024-12-02T21:16:04,524 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T21:16:04,526 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61949 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T21:16:04,526 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/C is initiating minor compaction (all files) 2024-12-02T21:16:04,526 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/C in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:04,526 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/e6e6da7058b94284890d5211a7d6f906, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/8c481ce10f62411a91a202724a47ee1c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/029499031b7c4b94899332f235c0dca3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f8981f9fa30f400988f74e541d51b304, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/d7a4cef320ce47dd96950bb246be9717] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=60.5 K 2024-12-02T21:16:04,526 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e6da7058b94284890d5211a7d6f906, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=754, earliestPutTs=1733174157779 2024-12-02T21:16:04,528 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c481ce10f62411a91a202724a47ee1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=791, earliestPutTs=1733174158078 2024-12-02T21:16:04,528 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/3018caf4cb204e0f82892f9079537176 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3018caf4cb204e0f82892f9079537176 2024-12-02T21:16:04,528 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 029499031b7c4b94899332f235c0dca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=804, earliestPutTs=1733174158575 2024-12-02T21:16:04,529 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f8981f9fa30f400988f74e541d51b304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=841, earliestPutTs=1733174160369 2024-12-02T21:16:04,529 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting d7a4cef320ce47dd96950bb246be9717, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=852, earliestPutTs=1733174162568 2024-12-02T21:16:04,533 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/A of ab928e240d6dc631659396e2f2d0cdda into 3018caf4cb204e0f82892f9079537176(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:04,533 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:04,533 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/A, priority=11, startTime=1733174164088; duration=0sec 2024-12-02T21:16:04,533 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:04,533 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:A 2024-12-02T21:16:04,539 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:04,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 415 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174224534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:04,540 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/f81dff0de38240f6a439cfe7b8d52831 is 50, key is test_row_0/C:col10/1733174162568/Put/seqid=0 2024-12-02T21:16:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:04,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742041_1217 (size=12915) 2024-12-02T21:16:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T21:16:04,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:04,630 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025d6c8827188042d7b5d512a9f4b184aa_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025d6c8827188042d7b5d512a9f4b184aa_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:04,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a2d1a37109d44811a83d3c041692b186, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:04,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a2d1a37109d44811a83d3c041692b186 is 175, key is test_row_0/A:col10/1733174163352/Put/seqid=0 2024-12-02T21:16:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742042_1218 (size=31255) 2024-12-02T21:16:04,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 416 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174224648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:04,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:04,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 417 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174224858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:04,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:04,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:04,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:04,967 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/f81dff0de38240f6a439cfe7b8d52831 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f81dff0de38240f6a439cfe7b8d52831 2024-12-02T21:16:04,977 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/C of ab928e240d6dc631659396e2f2d0cdda into f81dff0de38240f6a439cfe7b8d52831(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:04,977 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:04,977 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/C, priority=11, startTime=1733174164088; duration=0sec 2024-12-02T21:16:04,977 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:04,977 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:C 2024-12-02T21:16:05,035 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=886, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a2d1a37109d44811a83d3c041692b186 2024-12-02T21:16:05,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/901e8db5fd834f46ae449c6a5f8c9339 is 50, key is test_row_0/B:col10/1733174163352/Put/seqid=0 2024-12-02T21:16:05,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742043_1219 (size=12301) 2024-12-02T21:16:05,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 418 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174225169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:05,447 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/901e8db5fd834f46ae449c6a5f8c9339 2024-12-02T21:16:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/cd74cd20d5a44b5c91ce3255718c5341 is 50, key is test_row_0/C:col10/1733174163352/Put/seqid=0 2024-12-02T21:16:05,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742044_1220 (size=12301) 2024-12-02T21:16:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T21:16:05,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 419 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174225678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:05,859 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/cd74cd20d5a44b5c91ce3255718c5341 2024-12-02T21:16:05,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a2d1a37109d44811a83d3c041692b186 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a2d1a37109d44811a83d3c041692b186 2024-12-02T21:16:05,868 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a2d1a37109d44811a83d3c041692b186, entries=150, sequenceid=886, filesize=30.5 K 2024-12-02T21:16:05,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/901e8db5fd834f46ae449c6a5f8c9339 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/901e8db5fd834f46ae449c6a5f8c9339 2024-12-02T21:16:05,873 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/901e8db5fd834f46ae449c6a5f8c9339, entries=150, sequenceid=886, filesize=12.0 K 2024-12-02T21:16:05,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/cd74cd20d5a44b5c91ce3255718c5341 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/cd74cd20d5a44b5c91ce3255718c5341 2024-12-02T21:16:05,878 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/cd74cd20d5a44b5c91ce3255718c5341, entries=150, sequenceid=886, filesize=12.0 K 2024-12-02T21:16:05,879 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for ab928e240d6dc631659396e2f2d0cdda in 1668ms, sequenceid=886, compaction requested=false 2024-12-02T21:16:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-02T21:16:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-02T21:16:05,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-02T21:16:05,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4390 sec 2024-12-02T21:16:05,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.4440 sec 2024-12-02T21:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:06,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T21:16:06,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:16:06,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:06,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:16:06,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:06,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:16:06,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:06,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e5d1bdbff3144f46ab39e43328fc6f55_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174164532/Put/seqid=0 2024-12-02T21:16:06,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742045_1221 (size=14994) 2024-12-02T21:16:06,708 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:06,711 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e5d1bdbff3144f46ab39e43328fc6f55_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e5d1bdbff3144f46ab39e43328fc6f55_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:06,712 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/e615dba984354a0297493fb01f81579b, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:06,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/e615dba984354a0297493fb01f81579b is 175, key is test_row_0/A:col10/1733174164532/Put/seqid=0 2024-12-02T21:16:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742046_1222 (size=39949) 2024-12-02T21:16:06,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 438 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174226756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:06,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58248 deadline: 1733174226779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:06,781 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:06,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1733174226809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:06,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58234 deadline: 1733174226809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:06,811 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:06,811 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in ab928e240d6dc631659396e2f2d0cdda of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:06,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 439 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174226869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:06,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:07,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 440 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174227078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:07,117 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=901, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/e615dba984354a0297493fb01f81579b 2024-12-02T21:16:07,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/7dbb71ab6a8449d9a4a04a38413e638a is 50, key is test_row_0/B:col10/1733174164532/Put/seqid=0 2024-12-02T21:16:07,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742047_1223 (size=12301) 2024-12-02T21:16:07,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 441 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174227389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:07,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=901 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/7dbb71ab6a8449d9a4a04a38413e638a 2024-12-02T21:16:07,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/848ce417bd3444fe93bfec101eb34ee5 is 50, key is test_row_0/C:col10/1733174164532/Put/seqid=0 2024-12-02T21:16:07,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742048_1224 (size=12301) 2024-12-02T21:16:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T21:16:07,609 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:07,610 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-02T21:16:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T21:16:07,611 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:07,612 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:07,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T21:16:07,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T21:16:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:07,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:07,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 442 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58162 deadline: 1733174227898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:07,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T21:16:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:07,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T21:16:07,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=901 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/848ce417bd3444fe93bfec101eb34ee5 2024-12-02T21:16:07,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/e615dba984354a0297493fb01f81579b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e615dba984354a0297493fb01f81579b 2024-12-02T21:16:07,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e615dba984354a0297493fb01f81579b, entries=200, sequenceid=901, filesize=39.0 K 2024-12-02T21:16:07,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/7dbb71ab6a8449d9a4a04a38413e638a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7dbb71ab6a8449d9a4a04a38413e638a 2024-12-02T21:16:07,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7dbb71ab6a8449d9a4a04a38413e638a, entries=150, sequenceid=901, filesize=12.0 K 2024-12-02T21:16:07,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/848ce417bd3444fe93bfec101eb34ee5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/848ce417bd3444fe93bfec101eb34ee5 2024-12-02T21:16:07,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/848ce417bd3444fe93bfec101eb34ee5, entries=150, sequenceid=901, filesize=12.0 K 2024-12-02T21:16:07,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ab928e240d6dc631659396e2f2d0cdda in 1281ms, sequenceid=901, compaction requested=true 2024-12-02T21:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:07,973 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:07,973 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab928e240d6dc631659396e2f2d0cdda:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:07,975 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:07,975 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:07,975 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/B is initiating minor compaction (all files) 2024-12-02T21:16:07,975 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/A is initiating minor compaction (all files) 2024-12-02T21:16:07,975 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/A in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:07,975 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/B in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:07,976 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3018caf4cb204e0f82892f9079537176, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a2d1a37109d44811a83d3c041692b186, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e615dba984354a0297493fb01f81579b] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=100.7 K 2024-12-02T21:16:07,976 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6bc6567fd6bd4deb8014cfeb2d3a0e02, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/901e8db5fd834f46ae449c6a5f8c9339, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7dbb71ab6a8449d9a4a04a38413e638a] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=36.6 K 2024-12-02T21:16:07,976 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:07,976 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3018caf4cb204e0f82892f9079537176, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a2d1a37109d44811a83d3c041692b186, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e615dba984354a0297493fb01f81579b] 2024-12-02T21:16:07,976 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc6567fd6bd4deb8014cfeb2d3a0e02, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=852, earliestPutTs=1733174162568 2024-12-02T21:16:07,976 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3018caf4cb204e0f82892f9079537176, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=852, earliestPutTs=1733174162568 2024-12-02T21:16:07,976 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d1a37109d44811a83d3c041692b186, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1733174163342 2024-12-02T21:16:07,976 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 901e8db5fd834f46ae449c6a5f8c9339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1733174163342 2024-12-02T21:16:07,976 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e615dba984354a0297493fb01f81579b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=901, earliestPutTs=1733174164495 2024-12-02T21:16:07,976 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dbb71ab6a8449d9a4a04a38413e638a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=901, earliestPutTs=1733174164495 2024-12-02T21:16:07,982 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:07,983 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#B#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:07,984 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/610ca8e826bc4320b21d2ea01fa388bd is 50, key is test_row_0/B:col10/1733174164532/Put/seqid=0 2024-12-02T21:16:07,985 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412021a6afb0751f445629258f4ea9b041d3d_ab928e240d6dc631659396e2f2d0cdda store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:07,987 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412021a6afb0751f445629258f4ea9b041d3d_ab928e240d6dc631659396e2f2d0cdda, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:07,987 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021a6afb0751f445629258f4ea9b041d3d_ab928e240d6dc631659396e2f2d0cdda because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:07,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742050_1226 (size=13017) 2024-12-02T21:16:07,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742049_1225 (size=4469) 2024-12-02T21:16:07,995 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#A#compaction#189 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:07,996 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/845ab96f9dc14dcdbd8e413ff7d73875 is 175, key is test_row_0/A:col10/1733174164532/Put/seqid=0 2024-12-02T21:16:07,999 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/610ca8e826bc4320b21d2ea01fa388bd as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/610ca8e826bc4320b21d2ea01fa388bd 2024-12-02T21:16:08,006 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/B of ab928e240d6dc631659396e2f2d0cdda into 610ca8e826bc4320b21d2ea01fa388bd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:08,006 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:08,006 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/B, priority=13, startTime=1733174167973; duration=0sec 2024-12-02T21:16:08,006 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:08,006 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:B 2024-12-02T21:16:08,006 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:08,009 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:08,009 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): ab928e240d6dc631659396e2f2d0cdda/C is initiating minor compaction (all files) 2024-12-02T21:16:08,009 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab928e240d6dc631659396e2f2d0cdda/C in TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:08,009 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f81dff0de38240f6a439cfe7b8d52831, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/cd74cd20d5a44b5c91ce3255718c5341, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/848ce417bd3444fe93bfec101eb34ee5] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp, totalSize=36.6 K 2024-12-02T21:16:08,011 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f81dff0de38240f6a439cfe7b8d52831, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=852, earliestPutTs=1733174162568 2024-12-02T21:16:08,012 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting cd74cd20d5a44b5c91ce3255718c5341, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1733174163342 2024-12-02T21:16:08,012 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 848ce417bd3444fe93bfec101eb34ee5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=901, earliestPutTs=1733174164495 2024-12-02T21:16:08,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742051_1227 (size=31971) 2024-12-02T21:16:08,024 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab928e240d6dc631659396e2f2d0cdda#C#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:08,025 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/5683860d2a6944f7a22f74cee21cd54f is 50, key is test_row_0/C:col10/1733174164532/Put/seqid=0 2024-12-02T21:16:08,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742052_1228 (size=13017) 2024-12-02T21:16:08,068 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/5683860d2a6944f7a22f74cee21cd54f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/5683860d2a6944f7a22f74cee21cd54f 2024-12-02T21:16:08,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T21:16:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:08,070 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T21:16:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:16:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:16:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:16:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:08,077 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/C of ab928e240d6dc631659396e2f2d0cdda into 5683860d2a6944f7a22f74cee21cd54f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:08,077 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:08,077 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/C, priority=13, startTime=1733174167973; duration=0sec 2024-12-02T21:16:08,078 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:08,078 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:C 2024-12-02T21:16:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028caa12dd8e0f4fe4b4186ea010fe5b78_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174166751/Put/seqid=0 2024-12-02T21:16:08,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742053_1229 (size=12454) 2024-12-02T21:16:08,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:08,095 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028caa12dd8e0f4fe4b4186ea010fe5b78_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028caa12dd8e0f4fe4b4186ea010fe5b78_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:08,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a5d6cca8f6c448f986c64d6ea861bde2, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:08,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a5d6cca8f6c448f986c64d6ea861bde2 is 175, key is test_row_0/A:col10/1733174166751/Put/seqid=0 2024-12-02T21:16:08,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742054_1230 (size=31255) 2024-12-02T21:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T21:16:08,427 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/845ab96f9dc14dcdbd8e413ff7d73875 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/845ab96f9dc14dcdbd8e413ff7d73875 2024-12-02T21:16:08,433 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab928e240d6dc631659396e2f2d0cdda/A of ab928e240d6dc631659396e2f2d0cdda into 845ab96f9dc14dcdbd8e413ff7d73875(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:08,433 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:08,433 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., storeName=ab928e240d6dc631659396e2f2d0cdda/A, priority=13, startTime=1733174167973; duration=0sec 2024-12-02T21:16:08,433 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:08,433 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab928e240d6dc631659396e2f2d0cdda:A 2024-12-02T21:16:08,506 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=922, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a5d6cca8f6c448f986c64d6ea861bde2 2024-12-02T21:16:08,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/e5deffa5dda940c7ab18b1fffb617951 is 50, key is test_row_0/B:col10/1733174166751/Put/seqid=0 2024-12-02T21:16:08,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742055_1231 (size=12301) 2024-12-02T21:16:08,662 INFO [Thread-708 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-708. 2024-12-02T21:16:08,662 DEBUG [Thread-708 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:08,662 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,662 INFO [Thread-704 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-704. 2024-12-02T21:16:08,662 DEBUG [Thread-704 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:08,662 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,662 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:08,663 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:08,663 INFO [Thread-706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-706. 2024-12-02T21:16:08,663 DEBUG [Thread-706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:08,663 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,663 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:08,663 INFO [Thread-702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-702. 2024-12-02T21:16:08,663 DEBUG [Thread-702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:08,663 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,663 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T21:16:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:08,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. as already flushing 2024-12-02T21:16:08,911 INFO [Thread-693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-693. 2024-12-02T21:16:08,911 DEBUG [Thread-693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:08,911 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,911 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:08,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:08,926 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=922 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/e5deffa5dda940c7ab18b1fffb617951 2024-12-02T21:16:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/98f5a225abf54cdfa340f15112aa51e8 is 50, key is test_row_0/C:col10/1733174166751/Put/seqid=0 2024-12-02T21:16:08,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742056_1232 (size=12301) 2024-12-02T21:16:09,362 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=922 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/98f5a225abf54cdfa340f15112aa51e8 2024-12-02T21:16:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/a5d6cca8f6c448f986c64d6ea861bde2 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a5d6cca8f6c448f986c64d6ea861bde2 2024-12-02T21:16:09,379 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a5d6cca8f6c448f986c64d6ea861bde2, entries=150, sequenceid=922, filesize=30.5 K 2024-12-02T21:16:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/e5deffa5dda940c7ab18b1fffb617951 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/e5deffa5dda940c7ab18b1fffb617951 2024-12-02T21:16:09,384 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/e5deffa5dda940c7ab18b1fffb617951, entries=150, sequenceid=922, filesize=12.0 K 2024-12-02T21:16:09,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/98f5a225abf54cdfa340f15112aa51e8 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/98f5a225abf54cdfa340f15112aa51e8 2024-12-02T21:16:09,388 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/98f5a225abf54cdfa340f15112aa51e8, entries=150, sequenceid=922, filesize=12.0 K 2024-12-02T21:16:09,389 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for ab928e240d6dc631659396e2f2d0cdda in 1319ms, sequenceid=922, compaction requested=false 2024-12-02T21:16:09,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for ab928e240d6dc631659396e2f2d0cdda: 2024-12-02T21:16:09,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:09,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-02T21:16:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-02T21:16:09,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-02T21:16:09,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7790 sec 2024-12-02T21:16:09,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.7830 sec 2024-12-02T21:16:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T21:16:09,759 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:12,178 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T21:16:12,703 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-12-02T21:16:12,704 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:12,704 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:12,704 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:12,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:16,852 INFO [Thread-691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-691. 2024-12-02T21:16:16,852 DEBUG [Thread-691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:16,852 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:16,852 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:16,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:16,882 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-12-02T21:16:16,882 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-12-02T21:16:16,882 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:16,882 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:16,882 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:16,882 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:16,883 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:16,883 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:16,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:16,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 392 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 149 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 170 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3418 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3476 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1541 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4621 rows 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1534 2024-12-02T21:16:16,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4600 rows 2024-12-02T21:16:16,883 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:16:16,883 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:16:16,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:16,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:16,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:16,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T21:16:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T21:16:16,888 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174176888"}]},"ts":"1733174176888"} 2024-12-02T21:16:16,890 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T21:16:16,890 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T21:16:16,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T21:16:16,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, UNASSIGN}] 2024-12-02T21:16:16,894 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, UNASSIGN 2024-12-02T21:16:16,895 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ab928e240d6dc631659396e2f2d0cdda, regionState=CLOSING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:16,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, UNASSIGN because future has completed 2024-12-02T21:16:16,897 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:16:16,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:16:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T21:16:17,050 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:17,050 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:16:17,050 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing ab928e240d6dc631659396e2f2d0cdda, disabling compactions & flushes 2024-12-02T21:16:17,050 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:17,050 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:17,050 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. after waiting 0 ms 2024-12-02T21:16:17,050 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:17,050 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing ab928e240d6dc631659396e2f2d0cdda 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T21:16:17,050 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=A 2024-12-02T21:16:17,050 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:17,050 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=B 2024-12-02T21:16:17,051 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:17,051 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab928e240d6dc631659396e2f2d0cdda, store=C 2024-12-02T21:16:17,051 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:17,058 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027fa262a8ce8742b0a4ea43fbf5dd5589_ab928e240d6dc631659396e2f2d0cdda is 50, key is test_row_0/A:col10/1733174168909/Put/seqid=0 2024-12-02T21:16:17,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742057_1233 (size=9914) 2024-12-02T21:16:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T21:16:17,474 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:17,478 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027fa262a8ce8742b0a4ea43fbf5dd5589_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fa262a8ce8742b0a4ea43fbf5dd5589_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:17,479 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/0463a35bdfe04c2ca0ace2dde5a6498e, store: [table=TestAcidGuarantees family=A region=ab928e240d6dc631659396e2f2d0cdda] 2024-12-02T21:16:17,479 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/0463a35bdfe04c2ca0ace2dde5a6498e is 175, key is test_row_0/A:col10/1733174168909/Put/seqid=0 2024-12-02T21:16:17,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742058_1234 (size=22561) 2024-12-02T21:16:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T21:16:17,884 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=932, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/0463a35bdfe04c2ca0ace2dde5a6498e 2024-12-02T21:16:17,897 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/36aa021965c74858b668c89babda861a is 50, key is test_row_0/B:col10/1733174168909/Put/seqid=0 2024-12-02T21:16:17,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742059_1235 (size=9857) 2024-12-02T21:16:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T21:16:18,302 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=932 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/36aa021965c74858b668c89babda861a 2024-12-02T21:16:18,310 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/50a56574551b4094b4b9451ddb45d5f1 is 50, key is test_row_0/C:col10/1733174168909/Put/seqid=0 2024-12-02T21:16:18,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742060_1236 (size=9857) 2024-12-02T21:16:18,717 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=932 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/50a56574551b4094b4b9451ddb45d5f1 2024-12-02T21:16:18,771 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/A/0463a35bdfe04c2ca0ace2dde5a6498e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0463a35bdfe04c2ca0ace2dde5a6498e 2024-12-02T21:16:18,777 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0463a35bdfe04c2ca0ace2dde5a6498e, entries=100, sequenceid=932, filesize=22.0 K 2024-12-02T21:16:18,779 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/B/36aa021965c74858b668c89babda861a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/36aa021965c74858b668c89babda861a 2024-12-02T21:16:18,784 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/36aa021965c74858b668c89babda861a, entries=100, sequenceid=932, filesize=9.6 K 2024-12-02T21:16:18,786 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/.tmp/C/50a56574551b4094b4b9451ddb45d5f1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/50a56574551b4094b4b9451ddb45d5f1 2024-12-02T21:16:18,791 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/50a56574551b4094b4b9451ddb45d5f1, entries=100, sequenceid=932, filesize=9.6 K 2024-12-02T21:16:18,792 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ab928e240d6dc631659396e2f2d0cdda in 1742ms, sequenceid=932, compaction requested=true 2024-12-02T21:16:18,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e29d23c48e054cd18070a4ca199cf928, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1f7eae28b3ef4edb8e9baf4d4c5f8594, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/6ede9e5e941c45dc8043915891919462, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/b594264600ae47d4b999e05fdb08cf7b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/65ab8149700c4773b9d495ed62a6d394, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a3511caca2824287bd8964f9ce4bdff7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0c5005349f9b4580bcb55aff43739704, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39d0c42940514e879da51356e93c390f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8d1cdef2dcb34f03bc10f26d4204aaac, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/d8615daf63ab4b81800926a36e627a6e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3c3b114fe4484fd6b66f8040b525f661, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8b88fdc493d1433d927c4bcb933aa33e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/04acd926ff8540ad90248b78e17bbfa2, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8a7301d4f40e48d5aa9ce74853aa5ae6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1a20a44d63a94d5a8193546c02870e5a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39e7903fc4f94298bbf86467d2dba02c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/f7b9b90b1c1743069fe565587277c5b1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3018caf4cb204e0f82892f9079537176, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/014df2799dbd467c9efc0b071bc77039, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a2d1a37109d44811a83d3c041692b186, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e615dba984354a0297493fb01f81579b] to archive 2024-12-02T21:16:18,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:16:18,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e29d23c48e054cd18070a4ca199cf928 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e29d23c48e054cd18070a4ca199cf928 2024-12-02T21:16:18,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1f7eae28b3ef4edb8e9baf4d4c5f8594 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1f7eae28b3ef4edb8e9baf4d4c5f8594 2024-12-02T21:16:18,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/6ede9e5e941c45dc8043915891919462 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/6ede9e5e941c45dc8043915891919462 2024-12-02T21:16:18,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/b594264600ae47d4b999e05fdb08cf7b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/b594264600ae47d4b999e05fdb08cf7b 2024-12-02T21:16:18,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/65ab8149700c4773b9d495ed62a6d394 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/65ab8149700c4773b9d495ed62a6d394 2024-12-02T21:16:18,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a3511caca2824287bd8964f9ce4bdff7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a3511caca2824287bd8964f9ce4bdff7 2024-12-02T21:16:18,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0c5005349f9b4580bcb55aff43739704 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0c5005349f9b4580bcb55aff43739704 2024-12-02T21:16:18,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39d0c42940514e879da51356e93c390f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39d0c42940514e879da51356e93c390f 2024-12-02T21:16:18,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8d1cdef2dcb34f03bc10f26d4204aaac to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8d1cdef2dcb34f03bc10f26d4204aaac 2024-12-02T21:16:18,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/d8615daf63ab4b81800926a36e627a6e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/d8615daf63ab4b81800926a36e627a6e 2024-12-02T21:16:18,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3c3b114fe4484fd6b66f8040b525f661 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3c3b114fe4484fd6b66f8040b525f661 2024-12-02T21:16:18,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8b88fdc493d1433d927c4bcb933aa33e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8b88fdc493d1433d927c4bcb933aa33e 2024-12-02T21:16:18,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/04acd926ff8540ad90248b78e17bbfa2 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/04acd926ff8540ad90248b78e17bbfa2 2024-12-02T21:16:18,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8a7301d4f40e48d5aa9ce74853aa5ae6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/8a7301d4f40e48d5aa9ce74853aa5ae6 2024-12-02T21:16:18,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1a20a44d63a94d5a8193546c02870e5a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/1a20a44d63a94d5a8193546c02870e5a 2024-12-02T21:16:18,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39e7903fc4f94298bbf86467d2dba02c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/39e7903fc4f94298bbf86467d2dba02c 2024-12-02T21:16:18,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/f7b9b90b1c1743069fe565587277c5b1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/f7b9b90b1c1743069fe565587277c5b1 2024-12-02T21:16:18,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3018caf4cb204e0f82892f9079537176 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/3018caf4cb204e0f82892f9079537176 2024-12-02T21:16:18,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/014df2799dbd467c9efc0b071bc77039 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/014df2799dbd467c9efc0b071bc77039 2024-12-02T21:16:18,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a2d1a37109d44811a83d3c041692b186 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a2d1a37109d44811a83d3c041692b186 2024-12-02T21:16:18,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e615dba984354a0297493fb01f81579b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/e615dba984354a0297493fb01f81579b 2024-12-02T21:16:18,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7a789c836f6b40d586b0b5cbf81954d5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/876a8b00117e4e3d8b1c57b8858341e7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4cedd3beeb3d4f0597757eb94b286ead, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b65b1fdbb47f4570a5c5a8051a4ab107, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/c204d08751af41bc932d06fc2b3e36f1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4ecf5a42d974454990dc0be335576c3e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/78aa4cdb8b3b41a3b149b56edb83e2c5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4829332c41e6493abc73d131c3e52dd2, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/23973d9992454b7d87a1b800aac92f76, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bd7a46e0348c41acba65e071e0cca8ea, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/a813e7f08ae3465e8437bac0c65830af, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/3242ab0857c044d5b4b6ab220a9e4dab, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/9ae51c04f09545c299868f72ad6dc617, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bc1b9188dbc14f7b9d6a5dea86e608b7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6baf37b889a84023a6eae588111b381e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/08ef853cd8d84f57a5e9794e37783b18, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b6e8a8b82c6640329f58fb01730e9063, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6bc6567fd6bd4deb8014cfeb2d3a0e02, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/da1c950936e345cbb3cd659041b991c3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/901e8db5fd834f46ae449c6a5f8c9339, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7dbb71ab6a8449d9a4a04a38413e638a] to archive 2024-12-02T21:16:18,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:16:18,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7a789c836f6b40d586b0b5cbf81954d5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7a789c836f6b40d586b0b5cbf81954d5 2024-12-02T21:16:18,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/876a8b00117e4e3d8b1c57b8858341e7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/876a8b00117e4e3d8b1c57b8858341e7 2024-12-02T21:16:18,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4cedd3beeb3d4f0597757eb94b286ead to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4cedd3beeb3d4f0597757eb94b286ead 2024-12-02T21:16:18,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b65b1fdbb47f4570a5c5a8051a4ab107 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b65b1fdbb47f4570a5c5a8051a4ab107 2024-12-02T21:16:18,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/c204d08751af41bc932d06fc2b3e36f1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/c204d08751af41bc932d06fc2b3e36f1 2024-12-02T21:16:18,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4ecf5a42d974454990dc0be335576c3e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4ecf5a42d974454990dc0be335576c3e 2024-12-02T21:16:18,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/78aa4cdb8b3b41a3b149b56edb83e2c5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/78aa4cdb8b3b41a3b149b56edb83e2c5 2024-12-02T21:16:18,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4829332c41e6493abc73d131c3e52dd2 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/4829332c41e6493abc73d131c3e52dd2 2024-12-02T21:16:18,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/23973d9992454b7d87a1b800aac92f76 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/23973d9992454b7d87a1b800aac92f76 2024-12-02T21:16:18,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bd7a46e0348c41acba65e071e0cca8ea to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bd7a46e0348c41acba65e071e0cca8ea 2024-12-02T21:16:18,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/a813e7f08ae3465e8437bac0c65830af to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/a813e7f08ae3465e8437bac0c65830af 2024-12-02T21:16:18,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/3242ab0857c044d5b4b6ab220a9e4dab to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/3242ab0857c044d5b4b6ab220a9e4dab 2024-12-02T21:16:18,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/9ae51c04f09545c299868f72ad6dc617 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/9ae51c04f09545c299868f72ad6dc617 2024-12-02T21:16:18,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bc1b9188dbc14f7b9d6a5dea86e608b7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/bc1b9188dbc14f7b9d6a5dea86e608b7 2024-12-02T21:16:18,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6baf37b889a84023a6eae588111b381e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6baf37b889a84023a6eae588111b381e 2024-12-02T21:16:18,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/08ef853cd8d84f57a5e9794e37783b18 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/08ef853cd8d84f57a5e9794e37783b18 2024-12-02T21:16:18,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b6e8a8b82c6640329f58fb01730e9063 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/b6e8a8b82c6640329f58fb01730e9063 2024-12-02T21:16:18,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6bc6567fd6bd4deb8014cfeb2d3a0e02 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/6bc6567fd6bd4deb8014cfeb2d3a0e02 2024-12-02T21:16:18,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/da1c950936e345cbb3cd659041b991c3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/da1c950936e345cbb3cd659041b991c3 2024-12-02T21:16:18,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/901e8db5fd834f46ae449c6a5f8c9339 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/901e8db5fd834f46ae449c6a5f8c9339 2024-12-02T21:16:18,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7dbb71ab6a8449d9a4a04a38413e638a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/7dbb71ab6a8449d9a4a04a38413e638a 2024-12-02T21:16:18,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ed0acde0909b4114ace92cf72de52e6b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/3307da366a9241d48d45be3d0ec0027e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/5cd82b9718864275b4162f1ddeb32108, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/1d0dce0a1bee4921b1a1a9cbd5ffb903, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/83bbcd7c154f46bbbea67d5a4513e141, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/a6a6a7c31a24425fa8b3839ca1a2ba97, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/123dd432968645f58d675bc258ad0b0e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/888a2feea273403d8ccd6aee1c745daf, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ef1f099bf4804a29a244d3ac73c34b6b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/af1f6805911b490a8c062f0ed4a0fc36, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/7c9a5e74f5fb4c2ab7f861f5f8fff530, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/455d1256178f41fa80397c43da810fd9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/e6e6da7058b94284890d5211a7d6f906, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/68548fc8d4974e9c83ea42bde3178be4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/8c481ce10f62411a91a202724a47ee1c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/029499031b7c4b94899332f235c0dca3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f8981f9fa30f400988f74e541d51b304, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f81dff0de38240f6a439cfe7b8d52831, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/d7a4cef320ce47dd96950bb246be9717, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/cd74cd20d5a44b5c91ce3255718c5341, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/848ce417bd3444fe93bfec101eb34ee5] to archive 2024-12-02T21:16:18,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:16:18,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ed0acde0909b4114ace92cf72de52e6b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ed0acde0909b4114ace92cf72de52e6b 2024-12-02T21:16:18,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/3307da366a9241d48d45be3d0ec0027e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/3307da366a9241d48d45be3d0ec0027e 2024-12-02T21:16:18,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/5cd82b9718864275b4162f1ddeb32108 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/5cd82b9718864275b4162f1ddeb32108 2024-12-02T21:16:18,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/1d0dce0a1bee4921b1a1a9cbd5ffb903 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/1d0dce0a1bee4921b1a1a9cbd5ffb903 2024-12-02T21:16:18,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/83bbcd7c154f46bbbea67d5a4513e141 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/83bbcd7c154f46bbbea67d5a4513e141 2024-12-02T21:16:18,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/a6a6a7c31a24425fa8b3839ca1a2ba97 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/a6a6a7c31a24425fa8b3839ca1a2ba97 2024-12-02T21:16:18,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/123dd432968645f58d675bc258ad0b0e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/123dd432968645f58d675bc258ad0b0e 2024-12-02T21:16:18,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/888a2feea273403d8ccd6aee1c745daf to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/888a2feea273403d8ccd6aee1c745daf 2024-12-02T21:16:18,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ef1f099bf4804a29a244d3ac73c34b6b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/ef1f099bf4804a29a244d3ac73c34b6b 2024-12-02T21:16:18,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/af1f6805911b490a8c062f0ed4a0fc36 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/af1f6805911b490a8c062f0ed4a0fc36 2024-12-02T21:16:18,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/7c9a5e74f5fb4c2ab7f861f5f8fff530 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/7c9a5e74f5fb4c2ab7f861f5f8fff530 2024-12-02T21:16:18,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/455d1256178f41fa80397c43da810fd9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/455d1256178f41fa80397c43da810fd9 2024-12-02T21:16:18,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/e6e6da7058b94284890d5211a7d6f906 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/e6e6da7058b94284890d5211a7d6f906 2024-12-02T21:16:18,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/68548fc8d4974e9c83ea42bde3178be4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/68548fc8d4974e9c83ea42bde3178be4 2024-12-02T21:16:18,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/8c481ce10f62411a91a202724a47ee1c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/8c481ce10f62411a91a202724a47ee1c 2024-12-02T21:16:18,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/029499031b7c4b94899332f235c0dca3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/029499031b7c4b94899332f235c0dca3 2024-12-02T21:16:18,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f8981f9fa30f400988f74e541d51b304 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f8981f9fa30f400988f74e541d51b304 2024-12-02T21:16:18,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f81dff0de38240f6a439cfe7b8d52831 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/f81dff0de38240f6a439cfe7b8d52831 2024-12-02T21:16:18,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/d7a4cef320ce47dd96950bb246be9717 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/d7a4cef320ce47dd96950bb246be9717 2024-12-02T21:16:18,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/cd74cd20d5a44b5c91ce3255718c5341 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/cd74cd20d5a44b5c91ce3255718c5341 2024-12-02T21:16:18,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/848ce417bd3444fe93bfec101eb34ee5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/848ce417bd3444fe93bfec101eb34ee5 2024-12-02T21:16:18,867 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/recovered.edits/935.seqid, newMaxSeqId=935, maxSeqId=4 2024-12-02T21:16:18,868 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. 2024-12-02T21:16:18,868 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for ab928e240d6dc631659396e2f2d0cdda: Waiting for close lock at 1733174177050Running coprocessor pre-close hooks at 1733174177050Disabling compacts and flushes for region at 1733174177050Disabling writes for close at 1733174177050Obtaining lock to block concurrent updates at 1733174177050Preparing flush snapshotting stores in ab928e240d6dc631659396e2f2d0cdda at 1733174177050Finished memstore snapshotting TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733174177051 (+1 ms)Flushing stores of TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda. at 1733174177051Flushing ab928e240d6dc631659396e2f2d0cdda/A: creating writer at 1733174177051Flushing mob file ab928e240d6dc631659396e2f2d0cdda/A: appending metadata at 1733174177058 (+7 ms)Flushing mob file ab928e240d6dc631659396e2f2d0cdda/A: closing flushed file at 1733174177058Flushing ab928e240d6dc631659396e2f2d0cdda/A: appending metadata at 1733174177479 (+421 ms)Flushing ab928e240d6dc631659396e2f2d0cdda/A: closing flushed file at 1733174177479Flushing ab928e240d6dc631659396e2f2d0cdda/B: creating writer at 1733174177893 (+414 ms)Flushing ab928e240d6dc631659396e2f2d0cdda/B: appending metadata at 1733174177896 (+3 ms)Flushing ab928e240d6dc631659396e2f2d0cdda/B: closing flushed file at 1733174177896Flushing ab928e240d6dc631659396e2f2d0cdda/C: creating writer at 1733174178307 (+411 ms)Flushing ab928e240d6dc631659396e2f2d0cdda/C: appending metadata at 1733174178310 (+3 ms)Flushing ab928e240d6dc631659396e2f2d0cdda/C: closing flushed file at 1733174178310Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a7713f: reopening flushed file at 1733174178769 (+459 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8897a0f: reopening flushed file at 1733174178777 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62795d99: reopening flushed file at 1733174178784 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ab928e240d6dc631659396e2f2d0cdda in 1742ms, sequenceid=932, compaction requested=true at 1733174178792 (+8 ms)Writing region close event to WAL at 1733174178864 (+72 ms)Running coprocessor post-close hooks at 1733174178868 (+4 ms)Closed at 1733174178868 2024-12-02T21:16:18,870 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:18,871 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ab928e240d6dc631659396e2f2d0cdda, regionState=CLOSED 2024-12-02T21:16:18,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:16:18,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-02T21:16:18,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure ab928e240d6dc631659396e2f2d0cdda, server=87c3fdb6c570,35563,1733174113859 in 1.9760 sec 2024-12-02T21:16:18,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-02T21:16:18,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab928e240d6dc631659396e2f2d0cdda, UNASSIGN in 1.9810 sec 2024-12-02T21:16:18,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-02T21:16:18,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9850 sec 2024-12-02T21:16:18,879 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174178879"}]},"ts":"1733174178879"} 2024-12-02T21:16:18,881 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T21:16:18,881 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T21:16:18,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9960 sec 2024-12-02T21:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T21:16:19,039 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:19,039 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T21:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:19,041 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T21:16:19,042 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:19,044 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,047 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/recovered.edits] 2024-12-02T21:16:19,049 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0463a35bdfe04c2ca0ace2dde5a6498e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/0463a35bdfe04c2ca0ace2dde5a6498e 2024-12-02T21:16:19,050 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/845ab96f9dc14dcdbd8e413ff7d73875 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/845ab96f9dc14dcdbd8e413ff7d73875 2024-12-02T21:16:19,051 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a5d6cca8f6c448f986c64d6ea861bde2 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/A/a5d6cca8f6c448f986c64d6ea861bde2 2024-12-02T21:16:19,053 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/36aa021965c74858b668c89babda861a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/36aa021965c74858b668c89babda861a 2024-12-02T21:16:19,054 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/610ca8e826bc4320b21d2ea01fa388bd to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/610ca8e826bc4320b21d2ea01fa388bd 2024-12-02T21:16:19,055 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/e5deffa5dda940c7ab18b1fffb617951 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/B/e5deffa5dda940c7ab18b1fffb617951 2024-12-02T21:16:19,058 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/50a56574551b4094b4b9451ddb45d5f1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/50a56574551b4094b4b9451ddb45d5f1 2024-12-02T21:16:19,059 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/5683860d2a6944f7a22f74cee21cd54f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/5683860d2a6944f7a22f74cee21cd54f 2024-12-02T21:16:19,060 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/98f5a225abf54cdfa340f15112aa51e8 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/C/98f5a225abf54cdfa340f15112aa51e8 2024-12-02T21:16:19,063 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/recovered.edits/935.seqid to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda/recovered.edits/935.seqid 2024-12-02T21:16:19,063 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,063 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T21:16:19,064 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T21:16:19,064 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T21:16:19,068 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020150afe1ef014345b32668c2fe47cddd_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020150afe1ef014345b32668c2fe47cddd_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,068 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020613088f8f5046e29dde38bc3e99d76d_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020613088f8f5046e29dde38bc3e99d76d_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,069 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021a83804a5abd4eac9a8929bb29d427c2_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021a83804a5abd4eac9a8929bb29d427c2_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,070 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202439550b897264deb80b69314a9375119_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202439550b897264deb80b69314a9375119_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,071 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024484934fce81499196432e1dd97373a6_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024484934fce81499196432e1dd97373a6_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,072 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025d6c8827188042d7b5d512a9f4b184aa_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025d6c8827188042d7b5d512a9f4b184aa_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,073 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202631cb20fa5aa4e03b173f4d4b8b17d71_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202631cb20fa5aa4e03b173f4d4b8b17d71_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,074 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026b62b5b538c542a3bbc291a22d4b76f6_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026b62b5b538c542a3bbc291a22d4b76f6_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,075 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fa262a8ce8742b0a4ea43fbf5dd5589_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fa262a8ce8742b0a4ea43fbf5dd5589_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,076 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fcac0336c7e45bd8167cd8528fb2170_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fcac0336c7e45bd8167cd8528fb2170_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,077 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028caa12dd8e0f4fe4b4186ea010fe5b78_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028caa12dd8e0f4fe4b4186ea010fe5b78_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,078 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a2b776ff6e574b5e9a3f470419f7c644_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a2b776ff6e574b5e9a3f470419f7c644_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,079 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a7fbb6eeebe443af821dad3341a02326_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a7fbb6eeebe443af821dad3341a02326_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,079 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ca3123bb8a864a3d9e4a47ec2c06d7a0_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ca3123bb8a864a3d9e4a47ec2c06d7a0_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,081 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d10e7baf674b4e90a0ad5cd25602459b_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d10e7baf674b4e90a0ad5cd25602459b_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,082 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d7c8e7b7d26540c0a132709680cba646_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d7c8e7b7d26540c0a132709680cba646_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,083 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e1b0c1ced43641eb945d8263b16f4623_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e1b0c1ced43641eb945d8263b16f4623_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,084 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e5d1bdbff3144f46ab39e43328fc6f55_ab928e240d6dc631659396e2f2d0cdda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e5d1bdbff3144f46ab39e43328fc6f55_ab928e240d6dc631659396e2f2d0cdda 2024-12-02T21:16:19,085 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T21:16:19,087 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:19,090 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T21:16:19,092 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T21:16:19,093 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:19,093 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T21:16:19,094 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733174179093"}]},"ts":"9223372036854775807"} 2024-12-02T21:16:19,096 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T21:16:19,096 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ab928e240d6dc631659396e2f2d0cdda, NAME => 'TestAcidGuarantees,,1733174145405.ab928e240d6dc631659396e2f2d0cdda.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T21:16:19,097 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T21:16:19,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733174179097"}]},"ts":"9223372036854775807"} 2024-12-02T21:16:19,099 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T21:16:19,100 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:19,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-12-02T21:16:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T21:16:19,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T21:16:19,149 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:19,156 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1636387191_22 at /127.0.0.1:36198 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 565), ProcessCount=11 (was 11), AvailableMemoryMB=1952 (was 2761) 2024-12-02T21:16:19,167 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=1952 2024-12-02T21:16:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T21:16:19,168 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:16:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:19,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T21:16:19,170 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:19,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-12-02T21:16:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T21:16:19,171 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T21:16:19,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742061_1237 (size=1061) 2024-12-02T21:16:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T21:16:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T21:16:19,596 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0 2024-12-02T21:16:19,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742062_1238 (size=53) 2024-12-02T21:16:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T21:16:20,006 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:16:20,006 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 744ddead248cd8e4726f3c3f3c9ab799, disabling compactions & flushes 2024-12-02T21:16:20,006 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:20,006 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:20,006 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. after waiting 0 ms 2024-12-02T21:16:20,006 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:20,007 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:20,007 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 744ddead248cd8e4726f3c3f3c9ab799: Waiting for close lock at 1733174180006Disabling compacts and flushes for region at 1733174180006Disabling writes for close at 1733174180006Writing region close event to WAL at 1733174180006Closed at 1733174180006 2024-12-02T21:16:20,008 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T21:16:20,009 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733174180009"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174180009"}]},"ts":"1733174180009"} 2024-12-02T21:16:20,012 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T21:16:20,014 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T21:16:20,014 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174180014"}]},"ts":"1733174180014"} 2024-12-02T21:16:20,016 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T21:16:20,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744ddead248cd8e4726f3c3f3c9ab799, ASSIGN}] 2024-12-02T21:16:20,019 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744ddead248cd8e4726f3c3f3c9ab799, ASSIGN 2024-12-02T21:16:20,020 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=744ddead248cd8e4726f3c3f3c9ab799, ASSIGN; state=OFFLINE, location=87c3fdb6c570,35563,1733174113859; forceNewPlan=false, retain=false 2024-12-02T21:16:20,170 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=744ddead248cd8e4726f3c3f3c9ab799, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:20,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=744ddead248cd8e4726f3c3f3c9ab799, ASSIGN because future has completed 2024-12-02T21:16:20,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:16:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T21:16:20,330 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:20,330 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:16:20,331 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,331 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:16:20,331 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,331 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,333 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,334 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:16:20,335 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 744ddead248cd8e4726f3c3f3c9ab799 columnFamilyName A 2024-12-02T21:16:20,335 DEBUG [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:20,335 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] regionserver.HStore(327): Store=744ddead248cd8e4726f3c3f3c9ab799/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:16:20,335 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,336 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:16:20,337 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 744ddead248cd8e4726f3c3f3c9ab799 columnFamilyName B 2024-12-02T21:16:20,337 DEBUG [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:20,337 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] regionserver.HStore(327): Store=744ddead248cd8e4726f3c3f3c9ab799/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:16:20,337 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,338 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:16:20,338 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 744ddead248cd8e4726f3c3f3c9ab799 columnFamilyName C 2024-12-02T21:16:20,338 DEBUG [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:20,338 INFO [StoreOpener-744ddead248cd8e4726f3c3f3c9ab799-1 {}] regionserver.HStore(327): Store=744ddead248cd8e4726f3c3f3c9ab799/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:16:20,338 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:20,338 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,339 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,339 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,340 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,340 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,340 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:16:20,341 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,343 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:16:20,344 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 744ddead248cd8e4726f3c3f3c9ab799; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62386378, jitterRate=-0.07037052512168884}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:16:20,344 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:20,344 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 744ddead248cd8e4726f3c3f3c9ab799: Running coprocessor pre-open hook at 1733174180331Writing region info on filesystem at 1733174180331Initializing all the Stores at 1733174180332 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174180332Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174180332Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174180332Cleaning up temporary data from old regions at 1733174180340 (+8 ms)Running coprocessor post-open hooks at 1733174180344 (+4 ms)Region opened successfully at 1733174180344 2024-12-02T21:16:20,345 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., pid=63, masterSystemTime=1733174180324 2024-12-02T21:16:20,347 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:20,347 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:20,347 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=744ddead248cd8e4726f3c3f3c9ab799, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:20,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:16:20,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-02T21:16:20,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 in 178 msec 2024-12-02T21:16:20,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-02T21:16:20,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744ddead248cd8e4726f3c3f3c9ab799, ASSIGN in 334 msec 2024-12-02T21:16:20,353 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T21:16:20,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174180353"}]},"ts":"1733174180353"} 2024-12-02T21:16:20,354 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T21:16:20,355 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T21:16:20,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-12-02T21:16:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T21:16:21,320 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:21,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,322 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be2e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,324 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e60fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,326 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,329 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T21:16:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 87c3fdb6c570,39577,1733174113031 2024-12-02T21:16:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@448f22f3 2024-12-02T21:16:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T21:16:21,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T21:16:21,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,333 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5561b01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,335 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325249d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,336 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da78ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,337 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b6f5f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,339 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,339 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444dc828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,340 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a826b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,342 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b73085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,343 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@285af562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,345 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23830ff2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,347 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fce2092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,349 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a8544f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,351 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d66c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,353 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a8c459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,354 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56cf5eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,356 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b18ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,358 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e76b18a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,360 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69801e5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,361 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8958f98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:21,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:21,364 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3130a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:21,365 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf23a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:21,366 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,366 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,366 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,367 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,367 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,367 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,369 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:21,369 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,369 DEBUG [Thread-1111 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,369 DEBUG [Thread-1111 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-02T21:16:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T21:16:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,373 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,374 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:21,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:21,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:16:21,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:21,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:21,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:21,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:21,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:21,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:21,384 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,384 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,388 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:21,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:21,390 DEBUG [Thread-1117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:16:21,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/fc5e145bd3be495f981035883b90980a is 50, key is test_row_0/A:col10/1733174181376/Put/seqid=0 2024-12-02T21:16:21,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174181397, max timestamp=1733174181417Num uniques 859; 2024-12-02T21:16:21,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:21,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174181397, max timestamp=1733174181417Num uniques 859; 2024-12-02T21:16:21,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:21,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174181397, max timestamp=1733174181417Num uniques 859; 2024-12-02T21:16:21,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:21,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742063_1239 (size=12001) 2024-12-02T21:16:21,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/fc5e145bd3be495f981035883b90980a 2024-12-02T21:16:21,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174241444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174241444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174241446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174241454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174241454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T21:16:21,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/4fb6f4c0e7da46dabfa2c7daf670a8be is 50, key is test_row_0/B:col10/1733174181376/Put/seqid=0 2024-12-02T21:16:21,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742064_1240 (size=12001) 2024-12-02T21:16:21,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T21:16:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:21,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174241558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174241559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174241559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174241569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174241569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T21:16:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:21,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T21:16:21,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174241769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174241769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174241769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174241778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174241778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:21,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T21:16:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:21,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/4fb6f4c0e7da46dabfa2c7daf670a8be 2024-12-02T21:16:21,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/e54f4844223f43f9bb1bbcbaee97fef7 is 50, key is test_row_0/C:col10/1733174181376/Put/seqid=0 2024-12-02T21:16:21,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T21:16:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:21,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:21,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742065_1241 (size=12001) 2024-12-02T21:16:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T21:16:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174242079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174242079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174242079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174242089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174242089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T21:16:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:22,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:22,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:22,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T21:16:22,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:22,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:22,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:22,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:22,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:22,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/e54f4844223f43f9bb1bbcbaee97fef7 2024-12-02T21:16:22,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/fc5e145bd3be495f981035883b90980a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fc5e145bd3be495f981035883b90980a 2024-12-02T21:16:22,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fc5e145bd3be495f981035883b90980a, entries=150, sequenceid=12, filesize=11.7 K 2024-12-02T21:16:22,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/4fb6f4c0e7da46dabfa2c7daf670a8be as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/4fb6f4c0e7da46dabfa2c7daf670a8be 2024-12-02T21:16:22,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/4fb6f4c0e7da46dabfa2c7daf670a8be, entries=150, sequenceid=12, filesize=11.7 K 2024-12-02T21:16:22,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/e54f4844223f43f9bb1bbcbaee97fef7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e54f4844223f43f9bb1bbcbaee97fef7 2024-12-02T21:16:22,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e54f4844223f43f9bb1bbcbaee97fef7, entries=150, sequenceid=12, filesize=11.7 K 2024-12-02T21:16:22,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 744ddead248cd8e4726f3c3f3c9ab799 in 1036ms, sequenceid=12, compaction requested=false 2024-12-02T21:16:22,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:22,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T21:16:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:22,453 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T21:16:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:22,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:22,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:22,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/60cbe57e71ba4e22a3e41cc1334eeb7d is 50, key is test_row_0/A:col10/1733174181451/Put/seqid=0 2024-12-02T21:16:22,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742066_1242 (size=12001) 2024-12-02T21:16:22,471 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/60cbe57e71ba4e22a3e41cc1334eeb7d 2024-12-02T21:16:22,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/e52c742dfeb94f54821c89c8715225e9 is 50, key is test_row_0/B:col10/1733174181451/Put/seqid=0 2024-12-02T21:16:22,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742067_1243 (size=12001) 2024-12-02T21:16:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T21:16:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:22,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:22,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174242596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174242599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174242599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174242599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174242599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174242709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174242709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174242709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,889 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/e52c742dfeb94f54821c89c8715225e9 2024-12-02T21:16:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/f4d7ddd4955f41ebb65e635abf7a9331 is 50, key is test_row_0/C:col10/1733174181451/Put/seqid=0 2024-12-02T21:16:22,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742068_1244 (size=12001) 2024-12-02T21:16:22,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174242919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174242919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174242919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174243228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174243228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174243229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,301 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/f4d7ddd4955f41ebb65e635abf7a9331 2024-12-02T21:16:23,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/60cbe57e71ba4e22a3e41cc1334eeb7d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/60cbe57e71ba4e22a3e41cc1334eeb7d 2024-12-02T21:16:23,309 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/60cbe57e71ba4e22a3e41cc1334eeb7d, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T21:16:23,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/e52c742dfeb94f54821c89c8715225e9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e52c742dfeb94f54821c89c8715225e9 2024-12-02T21:16:23,313 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e52c742dfeb94f54821c89c8715225e9, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T21:16:23,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/f4d7ddd4955f41ebb65e635abf7a9331 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f4d7ddd4955f41ebb65e635abf7a9331 2024-12-02T21:16:23,318 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f4d7ddd4955f41ebb65e635abf7a9331, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T21:16:23,318 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 744ddead248cd8e4726f3c3f3c9ab799 in 865ms, sequenceid=49, compaction requested=false 2024-12-02T21:16:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:23,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:23,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-02T21:16:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-02T21:16:23,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-02T21:16:23,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9460 sec 2024-12-02T21:16:23,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.9530 sec 2024-12-02T21:16:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T21:16:23,519 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:23,519 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-02T21:16:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T21:16:23,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:23,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:23,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:23,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T21:16:23,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:23,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:23,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:23,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:23,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:23,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:23,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/195675cc92a543488cdda16a48072af9 is 50, key is test_row_0/A:col10/1733174182591/Put/seqid=0 2024-12-02T21:16:23,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742069_1245 (size=12001) 2024-12-02T21:16:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T21:16:23,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174183628, max timestamp=1733174183648Num uniques 859; 2024-12-02T21:16:23,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:23,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174183628, max timestamp=1733174183648Num uniques 859; 2024-12-02T21:16:23,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:23,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733174183628, max timestamp=1733174183648Num uniques 859; 2024-12-02T21:16:23,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:23,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T21:16:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:23,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:23,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174243683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174243683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174243739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174243739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174243739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174243799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174243799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T21:16:23,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T21:16:23,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:23,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:23,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:23,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:23,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:23,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T21:16:23,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:23,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:23,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:23,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:23,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174244008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:24,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174244008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:24,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/195675cc92a543488cdda16a48072af9 2024-12-02T21:16:24,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/0dc8fa2fd02840019583fed09a2f8fa5 is 50, key is test_row_0/B:col10/1733174182591/Put/seqid=0 2024-12-02T21:16:24,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742070_1246 (size=12001) 2024-12-02T21:16:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T21:16:24,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T21:16:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T21:16:24,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:24,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174244319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:24,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174244319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:24,399 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T21:16:24,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/0dc8fa2fd02840019583fed09a2f8fa5 2024-12-02T21:16:24,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/4252b9725d404aa195e4e196040b7917 is 50, key is test_row_0/C:col10/1733174182591/Put/seqid=0 2024-12-02T21:16:24,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742071_1247 (size=12001) 2024-12-02T21:16:24,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T21:16:24,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:24,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T21:16:24,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:24,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T21:16:24,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174244749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:24,752 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:24,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T21:16:24,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:24,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174244758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:24,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174244758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:24,761 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:24,764 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:24,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174244829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:24,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174244829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:24,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/4252b9725d404aa195e4e196040b7917 2024-12-02T21:16:24,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/195675cc92a543488cdda16a48072af9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/195675cc92a543488cdda16a48072af9 2024-12-02T21:16:24,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/195675cc92a543488cdda16a48072af9, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T21:16:24,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/0dc8fa2fd02840019583fed09a2f8fa5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/0dc8fa2fd02840019583fed09a2f8fa5 2024-12-02T21:16:24,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/0dc8fa2fd02840019583fed09a2f8fa5, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T21:16:24,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/4252b9725d404aa195e4e196040b7917 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4252b9725d404aa195e4e196040b7917 2024-12-02T21:16:24,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4252b9725d404aa195e4e196040b7917, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T21:16:24,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 744ddead248cd8e4726f3c3f3c9ab799 in 1254ms, sequenceid=61, compaction requested=true 2024-12-02T21:16:24,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:24,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:24,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:24,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:24,865 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:24,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:24,865 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:24,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:24,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:24,866 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:24,866 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:24,866 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/B is initiating minor compaction (all files) 2024-12-02T21:16:24,866 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/A is initiating minor compaction (all files) 2024-12-02T21:16:24,866 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/B in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,866 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/A in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,866 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fc5e145bd3be495f981035883b90980a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/60cbe57e71ba4e22a3e41cc1334eeb7d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/195675cc92a543488cdda16a48072af9] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.2 K 2024-12-02T21:16:24,866 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/4fb6f4c0e7da46dabfa2c7daf670a8be, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e52c742dfeb94f54821c89c8715225e9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/0dc8fa2fd02840019583fed09a2f8fa5] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.2 K 2024-12-02T21:16:24,867 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc5e145bd3be495f981035883b90980a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733174181376 2024-12-02T21:16:24,867 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb6f4c0e7da46dabfa2c7daf670a8be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733174181376 2024-12-02T21:16:24,867 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting e52c742dfeb94f54821c89c8715225e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733174181441 2024-12-02T21:16:24,867 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60cbe57e71ba4e22a3e41cc1334eeb7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733174181441 2024-12-02T21:16:24,867 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc8fa2fd02840019583fed09a2f8fa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733174182591 2024-12-02T21:16:24,867 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 195675cc92a543488cdda16a48072af9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733174182591 2024-12-02T21:16:24,875 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#A#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:24,875 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/54ded5732f3b4ce98e1b84b79dbf0fce is 50, key is test_row_0/A:col10/1733174182591/Put/seqid=0 2024-12-02T21:16:24,876 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#B#compaction#207 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:24,877 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/77601444eb1244159bd5e67261b86c36 is 50, key is test_row_0/B:col10/1733174182591/Put/seqid=0 2024-12-02T21:16:24,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742072_1248 (size=12104) 2024-12-02T21:16:24,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742073_1249 (size=12104) 2024-12-02T21:16:24,890 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/54ded5732f3b4ce98e1b84b79dbf0fce as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/54ded5732f3b4ce98e1b84b79dbf0fce 2024-12-02T21:16:24,896 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/A of 744ddead248cd8e4726f3c3f3c9ab799 into 54ded5732f3b4ce98e1b84b79dbf0fce(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:24,896 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:24,896 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/A, priority=13, startTime=1733174184865; duration=0sec 2024-12-02T21:16:24,896 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:24,896 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:A 2024-12-02T21:16:24,896 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:24,897 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:24,897 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/C is initiating minor compaction (all files) 2024-12-02T21:16:24,897 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/C in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,897 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e54f4844223f43f9bb1bbcbaee97fef7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f4d7ddd4955f41ebb65e635abf7a9331, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4252b9725d404aa195e4e196040b7917] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.2 K 2024-12-02T21:16:24,897 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e54f4844223f43f9bb1bbcbaee97fef7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733174181376 2024-12-02T21:16:24,898 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4d7ddd4955f41ebb65e635abf7a9331, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733174181441 2024-12-02T21:16:24,898 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4252b9725d404aa195e4e196040b7917, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733174182591 2024-12-02T21:16:24,904 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#C#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:24,905 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ee8b60517d174e4c952aa0f321f99e5d is 50, key is test_row_0/C:col10/1733174182591/Put/seqid=0 2024-12-02T21:16:24,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742074_1250 (size=12104) 2024-12-02T21:16:24,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T21:16:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,914 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-02T21:16:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/5eb31409b613454b841968b85b203767 is 50, key is test_row_0/A:col10/1733174183657/Put/seqid=0 2024-12-02T21:16:24,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742075_1251 (size=12001) 2024-12-02T21:16:24,923 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/5eb31409b613454b841968b85b203767 2024-12-02T21:16:24,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/33208cb6de334c14855723f44e4b2797 is 50, key is test_row_0/B:col10/1733174183657/Put/seqid=0 2024-12-02T21:16:24,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742076_1252 (size=12001) 2024-12-02T21:16:24,935 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/33208cb6de334c14855723f44e4b2797 2024-12-02T21:16:24,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/3741ee2c171f4b91b7171d63b6f9841e is 50, key is test_row_0/C:col10/1733174183657/Put/seqid=0 2024-12-02T21:16:24,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742077_1253 (size=12001) 2024-12-02T21:16:24,950 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/3741ee2c171f4b91b7171d63b6f9841e 2024-12-02T21:16:24,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/5eb31409b613454b841968b85b203767 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/5eb31409b613454b841968b85b203767 2024-12-02T21:16:24,957 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/5eb31409b613454b841968b85b203767, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T21:16:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/33208cb6de334c14855723f44e4b2797 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/33208cb6de334c14855723f44e4b2797 2024-12-02T21:16:24,962 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/33208cb6de334c14855723f44e4b2797, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T21:16:24,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/3741ee2c171f4b91b7171d63b6f9841e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/3741ee2c171f4b91b7171d63b6f9841e 2024-12-02T21:16:24,966 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/3741ee2c171f4b91b7171d63b6f9841e, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T21:16:24,967 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=0 B/0 for 744ddead248cd8e4726f3c3f3c9ab799 in 54ms, sequenceid=96, compaction requested=false 2024-12-02T21:16:24,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:24,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:24,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-02T21:16:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-02T21:16:24,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-02T21:16:24,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4480 sec 2024-12-02T21:16:24,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4520 sec 2024-12-02T21:16:25,294 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/77601444eb1244159bd5e67261b86c36 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/77601444eb1244159bd5e67261b86c36 2024-12-02T21:16:25,299 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/B of 744ddead248cd8e4726f3c3f3c9ab799 into 77601444eb1244159bd5e67261b86c36(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:25,299 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:25,299 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/B, priority=13, startTime=1733174184865; duration=0sec 2024-12-02T21:16:25,299 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:25,299 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:B 2024-12-02T21:16:25,315 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ee8b60517d174e4c952aa0f321f99e5d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ee8b60517d174e4c952aa0f321f99e5d 2024-12-02T21:16:25,320 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/C of 744ddead248cd8e4726f3c3f3c9ab799 into ee8b60517d174e4c952aa0f321f99e5d(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:25,320 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:25,320 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/C, priority=13, startTime=1733174184865; duration=0sec 2024-12-02T21:16:25,320 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:25,320 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:C 2024-12-02T21:16:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T21:16:25,669 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:25,669 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-02T21:16:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T21:16:25,671 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:25,672 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:25,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T21:16:25,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T21:16:25,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:25,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:25,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:25,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-02T21:16:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-02T21:16:25,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-02T21:16:25,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-12-02T21:16:25,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 162 msec 2024-12-02T21:16:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:25,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:16:25,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:25,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:25,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:25,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:25,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:25,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:25,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/2e5660c278494ecb8e3e4bb7b901a2a5 is 50, key is test_row_0/A:col10/1733174185849/Put/seqid=0 2024-12-02T21:16:25,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742078_1254 (size=12001) 2024-12-02T21:16:25,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/2e5660c278494ecb8e3e4bb7b901a2a5 2024-12-02T21:16:25,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/7be3499b979b40dbb34c9cdc27459dc4 is 50, key is test_row_0/B:col10/1733174185849/Put/seqid=0 2024-12-02T21:16:25,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733174185861, max timestamp=1733174185887Num uniques 859; 2024-12-02T21:16:25,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733174185861, max timestamp=1733174185887Num uniques 859; 2024-12-02T21:16:25,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:25,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:25,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733174185861, max timestamp=1733174185887Num uniques 859; 2024-12-02T21:16:25,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:25,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742079_1255 (size=12001) 2024-12-02T21:16:25,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174245920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174245924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T21:16:25,989 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:25,989 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-02T21:16:25,997 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T21:16:25,998 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:25,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:26,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174246029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174246039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T21:16:26,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T21:16:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174246239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174246249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/7be3499b979b40dbb34c9cdc27459dc4 2024-12-02T21:16:26,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/f8af32901a02454cb9812f267a817f94 is 50, key is test_row_0/C:col10/1733174185849/Put/seqid=0 2024-12-02T21:16:26,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T21:16:26,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:26,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742080_1256 (size=12001) 2024-12-02T21:16:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T21:16:26,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T21:16:26,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:26,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174246548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174246559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T21:16:26,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:26,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T21:16:26,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/f8af32901a02454cb9812f267a817f94 2024-12-02T21:16:26,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/2e5660c278494ecb8e3e4bb7b901a2a5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2e5660c278494ecb8e3e4bb7b901a2a5 2024-12-02T21:16:26,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2e5660c278494ecb8e3e4bb7b901a2a5, entries=150, sequenceid=109, filesize=11.7 K 2024-12-02T21:16:26,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/7be3499b979b40dbb34c9cdc27459dc4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7be3499b979b40dbb34c9cdc27459dc4 2024-12-02T21:16:26,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7be3499b979b40dbb34c9cdc27459dc4, entries=150, sequenceid=109, filesize=11.7 K 2024-12-02T21:16:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/f8af32901a02454cb9812f267a817f94 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f8af32901a02454cb9812f267a817f94 2024-12-02T21:16:26,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f8af32901a02454cb9812f267a817f94, entries=150, sequenceid=109, filesize=11.7 K 2024-12-02T21:16:26,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 744ddead248cd8e4726f3c3f3c9ab799 in 881ms, sequenceid=109, compaction requested=true 2024-12-02T21:16:26,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:26,732 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:26,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:26,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:26,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:26,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:26,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:26,733 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:26,734 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:26,734 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/B is initiating minor compaction (all files) 2024-12-02T21:16:26,735 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/B in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,735 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/77601444eb1244159bd5e67261b86c36, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/33208cb6de334c14855723f44e4b2797, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7be3499b979b40dbb34c9cdc27459dc4] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.3 K 2024-12-02T21:16:26,735 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:26,735 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/A is initiating minor compaction (all files) 2024-12-02T21:16:26,736 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/A in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,736 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/54ded5732f3b4ce98e1b84b79dbf0fce, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/5eb31409b613454b841968b85b203767, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2e5660c278494ecb8e3e4bb7b901a2a5] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.3 K 2024-12-02T21:16:26,736 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ded5732f3b4ce98e1b84b79dbf0fce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733174182591 2024-12-02T21:16:26,736 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77601444eb1244159bd5e67261b86c36, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733174182591 2024-12-02T21:16:26,737 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eb31409b613454b841968b85b203767, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733174183657 2024-12-02T21:16:26,737 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33208cb6de334c14855723f44e4b2797, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733174183657 2024-12-02T21:16:26,737 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e5660c278494ecb8e3e4bb7b901a2a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733174185845 2024-12-02T21:16:26,737 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7be3499b979b40dbb34c9cdc27459dc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733174185845 2024-12-02T21:16:26,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:26,749 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#B#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:26,749 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/e335074ab04b43faa39ad910fb1b1269 is 50, key is test_row_0/B:col10/1733174185849/Put/seqid=0 2024-12-02T21:16:26,754 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:26,755 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/a58e382f3e684f6484e54e6e7dde32e2 is 50, key is test_row_0/A:col10/1733174185849/Put/seqid=0 2024-12-02T21:16:26,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T21:16:26,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:26,762 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T21:16:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:26,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:26,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742082_1258 (size=12207) 2024-12-02T21:16:26,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/f240a59f007e494094f6c96aad432d2d is 50, key is test_row_0/A:col10/1733174185922/Put/seqid=0 2024-12-02T21:16:26,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174246783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174246785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174246785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742081_1257 (size=12207) 2024-12-02T21:16:26,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742083_1259 (size=12151) 2024-12-02T21:16:26,802 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/f240a59f007e494094f6c96aad432d2d 2024-12-02T21:16:26,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/7d4555d53df94d29b80c7fad2a131331 is 50, key is test_row_0/B:col10/1733174185922/Put/seqid=0 2024-12-02T21:16:26,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742084_1260 (size=12151) 2024-12-02T21:16:26,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174246899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174246899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174246899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174247058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174247068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174247109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174247109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174247109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T21:16:27,171 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/a58e382f3e684f6484e54e6e7dde32e2 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/a58e382f3e684f6484e54e6e7dde32e2 2024-12-02T21:16:27,175 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/A of 744ddead248cd8e4726f3c3f3c9ab799 into a58e382f3e684f6484e54e6e7dde32e2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:27,175 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:27,175 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/A, priority=13, startTime=1733174186732; duration=0sec 2024-12-02T21:16:27,175 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:27,175 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:A 2024-12-02T21:16:27,175 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:27,176 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:27,176 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/C is initiating minor compaction (all files) 2024-12-02T21:16:27,176 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/C in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:27,177 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ee8b60517d174e4c952aa0f321f99e5d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/3741ee2c171f4b91b7171d63b6f9841e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f8af32901a02454cb9812f267a817f94] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.3 K 2024-12-02T21:16:27,177 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ee8b60517d174e4c952aa0f321f99e5d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733174182591 2024-12-02T21:16:27,177 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 3741ee2c171f4b91b7171d63b6f9841e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733174183657 2024-12-02T21:16:27,177 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f8af32901a02454cb9812f267a817f94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733174185845 2024-12-02T21:16:27,184 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#C#compaction#219 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:27,184 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/e19cf643f2fe4254836f0d39fe0bc8a1 is 50, key is test_row_0/C:col10/1733174185849/Put/seqid=0 2024-12-02T21:16:27,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742085_1261 (size=12207) 2024-12-02T21:16:27,202 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/e335074ab04b43faa39ad910fb1b1269 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e335074ab04b43faa39ad910fb1b1269 2024-12-02T21:16:27,208 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/B of 744ddead248cd8e4726f3c3f3c9ab799 into e335074ab04b43faa39ad910fb1b1269(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:27,209 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:27,209 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/B, priority=13, startTime=1733174186732; duration=0sec 2024-12-02T21:16:27,209 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:27,209 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:B 2024-12-02T21:16:27,221 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/7d4555d53df94d29b80c7fad2a131331 2024-12-02T21:16:27,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/0ba8f534d9524995b976d0dfce567956 is 50, key is test_row_0/C:col10/1733174185922/Put/seqid=0 2024-12-02T21:16:27,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742086_1262 (size=12151) 2024-12-02T21:16:27,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174247419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174247419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174247419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,593 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/e19cf643f2fe4254836f0d39fe0bc8a1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e19cf643f2fe4254836f0d39fe0bc8a1 2024-12-02T21:16:27,597 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/C of 744ddead248cd8e4726f3c3f3c9ab799 into e19cf643f2fe4254836f0d39fe0bc8a1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:27,597 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:27,597 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/C, priority=13, startTime=1733174186732; duration=0sec 2024-12-02T21:16:27,598 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:27,598 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:C 2024-12-02T21:16:27,648 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/0ba8f534d9524995b976d0dfce567956 2024-12-02T21:16:27,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/f240a59f007e494094f6c96aad432d2d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f240a59f007e494094f6c96aad432d2d 2024-12-02T21:16:27,656 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f240a59f007e494094f6c96aad432d2d, entries=150, sequenceid=145, filesize=11.9 K 2024-12-02T21:16:27,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/7d4555d53df94d29b80c7fad2a131331 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7d4555d53df94d29b80c7fad2a131331 2024-12-02T21:16:27,661 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7d4555d53df94d29b80c7fad2a131331, entries=150, sequenceid=145, filesize=11.9 K 2024-12-02T21:16:27,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/0ba8f534d9524995b976d0dfce567956 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0ba8f534d9524995b976d0dfce567956 2024-12-02T21:16:27,665 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0ba8f534d9524995b976d0dfce567956, entries=150, sequenceid=145, filesize=11.9 K 2024-12-02T21:16:27,667 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 744ddead248cd8e4726f3c3f3c9ab799 in 904ms, sequenceid=145, compaction requested=false 2024-12-02T21:16:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-02T21:16:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-02T21:16:27,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-02T21:16:27,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6700 sec 2024-12-02T21:16:27,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6820 sec 2024-12-02T21:16:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:27,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T21:16:27,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:27,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:27,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:27,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:27,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:27,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:27,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/162b7c015a574ed498afa9ad719812e6 is 50, key is test_row_0/A:col10/1733174187930/Put/seqid=0 2024-12-02T21:16:27,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742087_1263 (size=16931) 2024-12-02T21:16:27,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/162b7c015a574ed498afa9ad719812e6 2024-12-02T21:16:27,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/cf6e05152201440f87c78be532322764 is 50, key is test_row_0/B:col10/1733174187930/Put/seqid=0 2024-12-02T21:16:27,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174247952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174247954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174247957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:27,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742088_1264 (size=12151) 2024-12-02T21:16:28,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174248059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174248069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174248069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174248078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174248078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,082 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:28,082 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:28,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T21:16:28,149 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:28,149 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-02T21:16:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T21:16:28,151 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:28,151 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:28,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T21:16:28,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174248269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174248278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174248278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T21:16:28,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/cf6e05152201440f87c78be532322764 2024-12-02T21:16:28,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/aa099a30b1c14f31a7a4fdbea086b044 is 50, key is test_row_0/C:col10/1733174187930/Put/seqid=0 2024-12-02T21:16:28,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742089_1265 (size=12151) 2024-12-02T21:16:28,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T21:16:28,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:28,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T21:16:28,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174248578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174248589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174248589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:28,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T21:16:28,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:28,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T21:16:28,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:28,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:28,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/aa099a30b1c14f31a7a4fdbea086b044 2024-12-02T21:16:28,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/162b7c015a574ed498afa9ad719812e6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/162b7c015a574ed498afa9ad719812e6 2024-12-02T21:16:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T21:16:28,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/162b7c015a574ed498afa9ad719812e6, entries=250, sequenceid=162, filesize=16.5 K 2024-12-02T21:16:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/cf6e05152201440f87c78be532322764 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cf6e05152201440f87c78be532322764 2024-12-02T21:16:28,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cf6e05152201440f87c78be532322764, entries=150, sequenceid=162, filesize=11.9 K 2024-12-02T21:16:28,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/aa099a30b1c14f31a7a4fdbea086b044 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/aa099a30b1c14f31a7a4fdbea086b044 2024-12-02T21:16:28,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/aa099a30b1c14f31a7a4fdbea086b044, entries=150, sequenceid=162, filesize=11.9 K 2024-12-02T21:16:28,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 744ddead248cd8e4726f3c3f3c9ab799 in 862ms, sequenceid=162, compaction requested=true 2024-12-02T21:16:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:28,792 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:28,793 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:28,793 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:28,793 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41289 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:28,794 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/A is initiating minor compaction (all files) 2024-12-02T21:16:28,794 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/B is initiating minor compaction (all files) 2024-12-02T21:16:28,794 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/A in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,794 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/a58e382f3e684f6484e54e6e7dde32e2, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f240a59f007e494094f6c96aad432d2d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/162b7c015a574ed498afa9ad719812e6] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=40.3 K 2024-12-02T21:16:28,794 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/B in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,794 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e335074ab04b43faa39ad910fb1b1269, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7d4555d53df94d29b80c7fad2a131331, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cf6e05152201440f87c78be532322764] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.7 K 2024-12-02T21:16:28,794 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting a58e382f3e684f6484e54e6e7dde32e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733174185845 2024-12-02T21:16:28,794 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting e335074ab04b43faa39ad910fb1b1269, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733174185845 2024-12-02T21:16:28,795 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f240a59f007e494094f6c96aad432d2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733174185913 2024-12-02T21:16:28,795 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d4555d53df94d29b80c7fad2a131331, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733174185913 2024-12-02T21:16:28,795 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 162b7c015a574ed498afa9ad719812e6, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733174186762 2024-12-02T21:16:28,795 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting cf6e05152201440f87c78be532322764, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733174187929 2024-12-02T21:16:28,803 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:28,803 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#A#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:28,803 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/f988f55d84d34c92bb641cc6b80bacc6 is 50, key is test_row_0/B:col10/1733174187930/Put/seqid=0 2024-12-02T21:16:28,803 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/4f6d10b3b3494405a57bdc4b45bc7c05 is 50, key is test_row_0/A:col10/1733174187930/Put/seqid=0 2024-12-02T21:16:28,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742090_1266 (size=12459) 2024-12-02T21:16:28,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742091_1267 (size=12459) 2024-12-02T21:16:28,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T21:16:28,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:28,917 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T21:16:28,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:28,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:28,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:28,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:28,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:28,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:28,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/fedfb46a303b4111b250f676d0c9253a is 50, key is test_row_0/A:col10/1733174187952/Put/seqid=0 2024-12-02T21:16:28,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742092_1268 (size=12151) 2024-12-02T21:16:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:29,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174249112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174249113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174249114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174249219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174249219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174249219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,235 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/4f6d10b3b3494405a57bdc4b45bc7c05 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/4f6d10b3b3494405a57bdc4b45bc7c05 2024-12-02T21:16:29,239 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/A of 744ddead248cd8e4726f3c3f3c9ab799 into 4f6d10b3b3494405a57bdc4b45bc7c05(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:29,239 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:29,239 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/A, priority=13, startTime=1733174188792; duration=0sec 2024-12-02T21:16:29,239 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:29,239 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:A 2024-12-02T21:16:29,240 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:29,240 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:29,240 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/C is initiating minor compaction (all files) 2024-12-02T21:16:29,240 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/C in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:29,241 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e19cf643f2fe4254836f0d39fe0bc8a1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0ba8f534d9524995b976d0dfce567956, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/aa099a30b1c14f31a7a4fdbea086b044] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.7 K 2024-12-02T21:16:29,241 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e19cf643f2fe4254836f0d39fe0bc8a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733174185845 2024-12-02T21:16:29,241 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ba8f534d9524995b976d0dfce567956, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733174185913 2024-12-02T21:16:29,241 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa099a30b1c14f31a7a4fdbea086b044, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733174187929 2024-12-02T21:16:29,247 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#C#compaction#227 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:29,247 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/80732d60c67e4740968c56c225ae2342 is 50, key is test_row_0/C:col10/1733174187930/Put/seqid=0 2024-12-02T21:16:29,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742093_1269 (size=12459) 2024-12-02T21:16:29,278 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/f988f55d84d34c92bb641cc6b80bacc6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/f988f55d84d34c92bb641cc6b80bacc6 2024-12-02T21:16:29,282 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/B of 744ddead248cd8e4726f3c3f3c9ab799 into f988f55d84d34c92bb641cc6b80bacc6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:29,282 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:29,282 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/B, priority=13, startTime=1733174188793; duration=0sec 2024-12-02T21:16:29,282 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:29,282 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:B 2024-12-02T21:16:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T21:16:29,326 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/fedfb46a303b4111b250f676d0c9253a 2024-12-02T21:16:29,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/cef03b1634294454b691bec345dfb718 is 50, key is test_row_0/B:col10/1733174187952/Put/seqid=0 2024-12-02T21:16:29,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742094_1270 (size=12151) 2024-12-02T21:16:29,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174249429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174249429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174249429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,654 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/80732d60c67e4740968c56c225ae2342 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/80732d60c67e4740968c56c225ae2342 2024-12-02T21:16:29,658 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/C of 744ddead248cd8e4726f3c3f3c9ab799 into 80732d60c67e4740968c56c225ae2342(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:29,658 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:29,659 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/C, priority=13, startTime=1733174188793; duration=0sec 2024-12-02T21:16:29,659 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:29,659 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:C 2024-12-02T21:16:29,737 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/cef03b1634294454b691bec345dfb718 2024-12-02T21:16:29,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174249738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174249738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174249738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:29,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/039d887d18bf4d0281c7a296cffa163a is 50, key is test_row_0/C:col10/1733174187952/Put/seqid=0 2024-12-02T21:16:29,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742095_1271 (size=12151) 2024-12-02T21:16:30,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174250089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,090 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174250098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,101 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,147 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/039d887d18bf4d0281c7a296cffa163a 2024-12-02T21:16:30,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/fedfb46a303b4111b250f676d0c9253a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fedfb46a303b4111b250f676d0c9253a 2024-12-02T21:16:30,162 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fedfb46a303b4111b250f676d0c9253a, entries=150, sequenceid=180, filesize=11.9 K 2024-12-02T21:16:30,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/cef03b1634294454b691bec345dfb718 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cef03b1634294454b691bec345dfb718 2024-12-02T21:16:30,168 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cef03b1634294454b691bec345dfb718, entries=150, sequenceid=180, filesize=11.9 K 2024-12-02T21:16:30,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/039d887d18bf4d0281c7a296cffa163a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/039d887d18bf4d0281c7a296cffa163a 2024-12-02T21:16:30,173 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/039d887d18bf4d0281c7a296cffa163a, entries=150, sequenceid=180, filesize=11.9 K 2024-12-02T21:16:30,174 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 744ddead248cd8e4726f3c3f3c9ab799 in 1258ms, sequenceid=180, compaction requested=false 2024-12-02T21:16:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-02T21:16:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-02T21:16:30,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-02T21:16:30,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-12-02T21:16:30,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.0270 sec 2024-12-02T21:16:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:30,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T21:16:30,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:30,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:30,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:30,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:30,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:30,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:30,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/6d21af1a159545adafdc32a253a1bc6a is 50, key is test_row_0/A:col10/1733174190250/Put/seqid=0 2024-12-02T21:16:30,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742096_1272 (size=14541) 2024-12-02T21:16:30,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174250265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174250266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174250269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T21:16:30,299 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:30,299 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-02T21:16:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T21:16:30,301 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:30,302 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:30,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:30,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174250378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174250379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174250379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T21:16:30,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T21:16:30,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:30,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:30,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:30,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174250589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174250589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174250589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T21:16:30,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:30,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:30,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:30,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T21:16:30,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/6d21af1a159545adafdc32a253a1bc6a 2024-12-02T21:16:30,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/6cd46e9b1b2846f5837af0a7be582331 is 50, key is test_row_0/B:col10/1733174190250/Put/seqid=0 2024-12-02T21:16:30,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742097_1273 (size=12151) 2024-12-02T21:16:30,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/6cd46e9b1b2846f5837af0a7be582331 2024-12-02T21:16:30,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/f55e1a9ecc244932ae55b1704db3d353 is 50, key is test_row_0/C:col10/1733174190250/Put/seqid=0 2024-12-02T21:16:30,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742098_1274 (size=12151) 2024-12-02T21:16:30,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T21:16:30,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:30,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:30,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:30,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174250898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174250898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174250898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:30,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T21:16:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:30,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:30,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:30,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T21:16:31,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T21:16:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:31,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:31,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/f55e1a9ecc244932ae55b1704db3d353 2024-12-02T21:16:31,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/6d21af1a159545adafdc32a253a1bc6a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6d21af1a159545adafdc32a253a1bc6a 2024-12-02T21:16:31,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6d21af1a159545adafdc32a253a1bc6a, entries=200, sequenceid=201, filesize=14.2 K 2024-12-02T21:16:31,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/6cd46e9b1b2846f5837af0a7be582331 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/6cd46e9b1b2846f5837af0a7be582331 2024-12-02T21:16:31,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/6cd46e9b1b2846f5837af0a7be582331, entries=150, sequenceid=201, filesize=11.9 K 2024-12-02T21:16:31,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/f55e1a9ecc244932ae55b1704db3d353 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f55e1a9ecc244932ae55b1704db3d353 2024-12-02T21:16:31,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f55e1a9ecc244932ae55b1704db3d353, entries=150, sequenceid=201, filesize=11.9 K 2024-12-02T21:16:31,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 744ddead248cd8e4726f3c3f3c9ab799 in 865ms, sequenceid=201, compaction requested=true 2024-12-02T21:16:31,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:31,117 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:31,117 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:31,118 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:31,118 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/A is initiating minor compaction (all files) 2024-12-02T21:16:31,118 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/A in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:31,118 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/4f6d10b3b3494405a57bdc4b45bc7c05, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fedfb46a303b4111b250f676d0c9253a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6d21af1a159545adafdc32a253a1bc6a] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=38.2 K 2024-12-02T21:16:31,118 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:31,118 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/B is initiating minor compaction (all files) 2024-12-02T21:16:31,118 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/B in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:31,118 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/f988f55d84d34c92bb641cc6b80bacc6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cef03b1634294454b691bec345dfb718, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/6cd46e9b1b2846f5837af0a7be582331] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.9 K 2024-12-02T21:16:31,118 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f988f55d84d34c92bb641cc6b80bacc6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733174187929 2024-12-02T21:16:31,118 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f6d10b3b3494405a57bdc4b45bc7c05, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733174187929 2024-12-02T21:16:31,119 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting cef03b1634294454b691bec345dfb718, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733174187951 2024-12-02T21:16:31,119 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting fedfb46a303b4111b250f676d0c9253a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733174187951 2024-12-02T21:16:31,119 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cd46e9b1b2846f5837af0a7be582331, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733174189111 2024-12-02T21:16:31,119 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d21af1a159545adafdc32a253a1bc6a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733174189111 2024-12-02T21:16:31,126 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:31,126 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/594bee5428494d9a8fb379f832441227 is 50, key is test_row_0/A:col10/1733174190250/Put/seqid=0 2024-12-02T21:16:31,137 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#B#compaction#234 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:31,138 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/d7e7b2a6dd724a289ef3323c026a4bee is 50, key is test_row_0/B:col10/1733174190250/Put/seqid=0 2024-12-02T21:16:31,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742099_1275 (size=12561) 2024-12-02T21:16:31,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742100_1276 (size=12561) 2024-12-02T21:16:31,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T21:16:31,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:31,219 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:16:31,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:31,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:31,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:31,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:31,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:31,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/9abbfb2bcc3141ee8243c4aff437cf34 is 50, key is test_row_0/A:col10/1733174190265/Put/seqid=0 2024-12-02T21:16:31,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742101_1277 (size=12151) 2024-12-02T21:16:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:31,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174251430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174251431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174251431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T21:16:31,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174251539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:31,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174251539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174251539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:31,547 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/594bee5428494d9a8fb379f832441227 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/594bee5428494d9a8fb379f832441227 2024-12-02T21:16:31,552 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/A of 744ddead248cd8e4726f3c3f3c9ab799 into 594bee5428494d9a8fb379f832441227(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:31,552 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:31,552 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/A, priority=13, startTime=1733174191116; duration=0sec 2024-12-02T21:16:31,552 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:31,552 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:A 2024-12-02T21:16:31,552 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:31,553 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:31,553 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/C is initiating minor compaction (all files) 2024-12-02T21:16:31,553 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/C in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:31,553 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/80732d60c67e4740968c56c225ae2342, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/039d887d18bf4d0281c7a296cffa163a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f55e1a9ecc244932ae55b1704db3d353] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=35.9 K 2024-12-02T21:16:31,554 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80732d60c67e4740968c56c225ae2342, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733174187929 2024-12-02T21:16:31,554 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039d887d18bf4d0281c7a296cffa163a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733174187951 2024-12-02T21:16:31,554 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55e1a9ecc244932ae55b1704db3d353, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733174189111 2024-12-02T21:16:31,560 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:31,561 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/64aa4d640094474087aeff21bd8e9790 is 50, key is test_row_0/C:col10/1733174190250/Put/seqid=0 2024-12-02T21:16:31,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742102_1278 (size=12561) 2024-12-02T21:16:31,568 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/d7e7b2a6dd724a289ef3323c026a4bee as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/d7e7b2a6dd724a289ef3323c026a4bee 2024-12-02T21:16:31,572 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/B of 744ddead248cd8e4726f3c3f3c9ab799 into d7e7b2a6dd724a289ef3323c026a4bee(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:31,572 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:31,572 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/B, priority=13, startTime=1733174191117; duration=0sec 2024-12-02T21:16:31,572 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:31,572 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:B 2024-12-02T21:16:31,626 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/9abbfb2bcc3141ee8243c4aff437cf34 2024-12-02T21:16:31,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/73174f9da2f34d87a15731d8299772e0 is 50, key is test_row_0/B:col10/1733174190265/Put/seqid=0 2024-12-02T21:16:31,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742103_1279 (size=12151) 2024-12-02T21:16:31,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:31,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174251749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174251749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:31,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174251749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:31,969 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/64aa4d640094474087aeff21bd8e9790 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/64aa4d640094474087aeff21bd8e9790 2024-12-02T21:16:31,974 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/C of 744ddead248cd8e4726f3c3f3c9ab799 into 64aa4d640094474087aeff21bd8e9790(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:31,974 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:31,974 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/C, priority=13, startTime=1733174191117; duration=0sec 2024-12-02T21:16:31,975 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:31,975 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:C 2024-12-02T21:16:32,040 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/73174f9da2f34d87a15731d8299772e0 2024-12-02T21:16:32,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/24ad8f773c444393817dcddbc471a5dc is 50, key is test_row_0/C:col10/1733174190265/Put/seqid=0 2024-12-02T21:16:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174252059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174252059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174252059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742104_1280 (size=12151) 2024-12-02T21:16:32,064 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/24ad8f773c444393817dcddbc471a5dc 2024-12-02T21:16:32,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/9abbfb2bcc3141ee8243c4aff437cf34 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/9abbfb2bcc3141ee8243c4aff437cf34 2024-12-02T21:16:32,072 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/9abbfb2bcc3141ee8243c4aff437cf34, entries=150, sequenceid=215, filesize=11.9 K 2024-12-02T21:16:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/73174f9da2f34d87a15731d8299772e0 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/73174f9da2f34d87a15731d8299772e0 2024-12-02T21:16:32,078 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/73174f9da2f34d87a15731d8299772e0, entries=150, sequenceid=215, filesize=11.9 K 2024-12-02T21:16:32,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/24ad8f773c444393817dcddbc471a5dc as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/24ad8f773c444393817dcddbc471a5dc 2024-12-02T21:16:32,084 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/24ad8f773c444393817dcddbc471a5dc, entries=150, sequenceid=215, filesize=11.9 K 2024-12-02T21:16:32,085 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 744ddead248cd8e4726f3c3f3c9ab799 in 866ms, sequenceid=215, compaction requested=false 2024-12-02T21:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-02T21:16:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-02T21:16:32,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-02T21:16:32,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7840 sec 2024-12-02T21:16:32,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7890 sec 2024-12-02T21:16:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T21:16:32,449 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:32,449 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-02T21:16:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T21:16:32,451 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:32,451 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:32,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T21:16:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:32,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=114.05 KB heapSize=339.89 KB 2024-12-02T21:16:32,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:32,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:32,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:32,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:32,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:32,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:32,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/3099e6ecce844f8fb310b597bbc79fc2 is 50, key is test_row_0/A:col10/1733174191430/Put/seqid=0 2024-12-02T21:16:32,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174252580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174252581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174252582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742105_1281 (size=12151) 2024-12-02T21:16:32,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:32,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:32,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:32,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:32,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:32,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174252689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174252689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174252689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:32,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:32,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T21:16:32,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174252899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174252899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174252899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:32,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:32,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:32,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:32,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:32,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:32,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:32,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/3099e6ecce844f8fb310b597bbc79fc2 2024-12-02T21:16:32,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/72cead9101a449859ae2120e3bcec0a1 is 50, key is test_row_0/B:col10/1733174191430/Put/seqid=0 2024-12-02T21:16:32,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742106_1282 (size=12151) 2024-12-02T21:16:33,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T21:16:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174253209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174253209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174253209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:33,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:33,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:33,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/72cead9101a449859ae2120e3bcec0a1 2024-12-02T21:16:33,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ed827d6a28d142908b042c4b4073d5f8 is 50, key is test_row_0/C:col10/1733174191430/Put/seqid=0 2024-12-02T21:16:33,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742107_1283 (size=12151) 2024-12-02T21:16:33,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T21:16:33,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:33,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174253719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174253719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:33,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174253719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:33,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ed827d6a28d142908b042c4b4073d5f8 2024-12-02T21:16:33,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/3099e6ecce844f8fb310b597bbc79fc2 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3099e6ecce844f8fb310b597bbc79fc2 2024-12-02T21:16:33,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3099e6ecce844f8fb310b597bbc79fc2, entries=150, sequenceid=239, filesize=11.9 K 2024-12-02T21:16:33,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/72cead9101a449859ae2120e3bcec0a1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/72cead9101a449859ae2120e3bcec0a1 2024-12-02T21:16:33,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/72cead9101a449859ae2120e3bcec0a1, entries=150, sequenceid=239, filesize=11.9 K 2024-12-02T21:16:33,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ed827d6a28d142908b042c4b4073d5f8 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ed827d6a28d142908b042c4b4073d5f8 2024-12-02T21:16:33,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ed827d6a28d142908b042c4b4073d5f8, entries=150, sequenceid=239, filesize=11.9 K 2024-12-02T21:16:33,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:33,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 744ddead248cd8e4726f3c3f3c9ab799 in 1254ms, sequenceid=239, compaction requested=true 2024-12-02T21:16:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:33,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:33,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,826 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:33,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:33,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:33,826 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:33,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:33,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:33,827 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:33,827 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:33,827 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/B is initiating minor compaction (all files) 2024-12-02T21:16:33,827 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/A is initiating minor compaction (all files) 2024-12-02T21:16:33,827 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/B in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,827 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/A in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,827 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/d7e7b2a6dd724a289ef3323c026a4bee, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/73174f9da2f34d87a15731d8299772e0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/72cead9101a449859ae2120e3bcec0a1] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=36.0 K 2024-12-02T21:16:33,827 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/594bee5428494d9a8fb379f832441227, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/9abbfb2bcc3141ee8243c4aff437cf34, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3099e6ecce844f8fb310b597bbc79fc2] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=36.0 K 2024-12-02T21:16:33,827 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 594bee5428494d9a8fb379f832441227, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733174189111 2024-12-02T21:16:33,827 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e7b2a6dd724a289ef3323c026a4bee, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733174189111 2024-12-02T21:16:33,828 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 73174f9da2f34d87a15731d8299772e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733174190264 2024-12-02T21:16:33,828 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9abbfb2bcc3141ee8243c4aff437cf34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733174190264 2024-12-02T21:16:33,828 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 72cead9101a449859ae2120e3bcec0a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733174191430 2024-12-02T21:16:33,828 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3099e6ecce844f8fb310b597bbc79fc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733174191430 2024-12-02T21:16:33,833 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:33,834 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/3344faf7e2e447fe97ed59135369743c is 50, key is test_row_0/A:col10/1733174191430/Put/seqid=0 2024-12-02T21:16:33,834 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:33,834 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/81649006a72049c888706089980756c8 is 50, key is test_row_0/B:col10/1733174191430/Put/seqid=0 2024-12-02T21:16:33,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742108_1284 (size=12663) 2024-12-02T21:16:33,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742109_1285 (size=12663) 2024-12-02T21:16:33,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T21:16:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:33,979 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:16:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:33,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/e02ee56497da4aba86dac4f0a0a2d40b is 50, key is test_row_0/A:col10/1733174192576/Put/seqid=0 2024-12-02T21:16:33,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742110_1286 (size=12151) 2024-12-02T21:16:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:34,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:34,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174254161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174194152, max timestamp=1733174194163Num uniques 859; 2024-12-02T21:16:34,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:34,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174194152, max timestamp=1733174194163Num uniques 859; 2024-12-02T21:16:34,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:34,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174194152, max timestamp=1733174194163Num uniques 859; 2024-12-02T21:16:34,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:34,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174254215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,242 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/3344faf7e2e447fe97ed59135369743c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3344faf7e2e447fe97ed59135369743c 2024-12-02T21:16:34,247 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/81649006a72049c888706089980756c8 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81649006a72049c888706089980756c8 2024-12-02T21:16:34,252 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/B of 744ddead248cd8e4726f3c3f3c9ab799 into 81649006a72049c888706089980756c8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:34,252 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:34,252 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/B, priority=13, startTime=1733174193826; duration=0sec 2024-12-02T21:16:34,252 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:34,252 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:B 2024-12-02T21:16:34,252 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:34,252 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/A of 744ddead248cd8e4726f3c3f3c9ab799 into 3344faf7e2e447fe97ed59135369743c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:34,253 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:34,253 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/A, priority=13, startTime=1733174193826; duration=0sec 2024-12-02T21:16:34,253 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:34,253 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:A 2024-12-02T21:16:34,253 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:34,253 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/C is initiating minor compaction (all files) 2024-12-02T21:16:34,253 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/C in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:34,253 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/64aa4d640094474087aeff21bd8e9790, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/24ad8f773c444393817dcddbc471a5dc, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ed827d6a28d142908b042c4b4073d5f8] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=36.0 K 2024-12-02T21:16:34,254 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 64aa4d640094474087aeff21bd8e9790, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733174189111 2024-12-02T21:16:34,254 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ad8f773c444393817dcddbc471a5dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733174190264 2024-12-02T21:16:34,254 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ed827d6a28d142908b042c4b4073d5f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733174191430 2024-12-02T21:16:34,261 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:34,261 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/cbce339d72994251a29f3e320543db05 is 50, key is test_row_0/C:col10/1733174191430/Put/seqid=0 2024-12-02T21:16:34,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742111_1287 (size=12663) 2024-12-02T21:16:34,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174254269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174254328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,387 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/e02ee56497da4aba86dac4f0a0a2d40b 2024-12-02T21:16:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/b10140cf4de54582b37fbb745aca25f7 is 50, key is test_row_0/B:col10/1733174192576/Put/seqid=0 2024-12-02T21:16:34,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742112_1288 (size=12151) 2024-12-02T21:16:34,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174254479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174254538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T21:16:34,669 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/cbce339d72994251a29f3e320543db05 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/cbce339d72994251a29f3e320543db05 2024-12-02T21:16:34,680 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/C of 744ddead248cd8e4726f3c3f3c9ab799 into cbce339d72994251a29f3e320543db05(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:34,680 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:34,680 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/C, priority=13, startTime=1733174193826; duration=0sec 2024-12-02T21:16:34,680 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:34,680 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:C 2024-12-02T21:16:34,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174254729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,730 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174254739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174254739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,742 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:34,742 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174254789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:34,796 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/b10140cf4de54582b37fbb745aca25f7 2024-12-02T21:16:34,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ca1c9e4a7fa34caa98b170c39a2dac92 is 50, key is test_row_0/C:col10/1733174192576/Put/seqid=0 2024-12-02T21:16:34,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742113_1289 (size=12151) 2024-12-02T21:16:34,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174254848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:34,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:35,207 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ca1c9e4a7fa34caa98b170c39a2dac92 2024-12-02T21:16:35,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/e02ee56497da4aba86dac4f0a0a2d40b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/e02ee56497da4aba86dac4f0a0a2d40b 2024-12-02T21:16:35,214 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/e02ee56497da4aba86dac4f0a0a2d40b, entries=150, sequenceid=250, filesize=11.9 K 2024-12-02T21:16:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/b10140cf4de54582b37fbb745aca25f7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/b10140cf4de54582b37fbb745aca25f7 2024-12-02T21:16:35,219 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/b10140cf4de54582b37fbb745aca25f7, entries=150, sequenceid=250, filesize=11.9 K 2024-12-02T21:16:35,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ca1c9e4a7fa34caa98b170c39a2dac92 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ca1c9e4a7fa34caa98b170c39a2dac92 2024-12-02T21:16:35,223 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ca1c9e4a7fa34caa98b170c39a2dac92, entries=150, sequenceid=250, filesize=11.9 K 2024-12-02T21:16:35,224 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 744ddead248cd8e4726f3c3f3c9ab799 in 1244ms, sequenceid=250, compaction requested=false 2024-12-02T21:16:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-02T21:16:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-02T21:16:35,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-02T21:16:35,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7740 sec 2024-12-02T21:16:35,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.7780 sec 2024-12-02T21:16:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:35,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T21:16:35,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:35,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:35,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:35,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:35,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:35,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:35,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/6c117199b95e4ac9bc527819f1763288 is 50, key is test_row_0/A:col10/1733174195300/Put/seqid=0 2024-12-02T21:16:35,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742114_1290 (size=14741) 2024-12-02T21:16:35,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174255328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:35,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174255358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:35,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174255439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:35,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174255649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:35,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/6c117199b95e4ac9bc527819f1763288 2024-12-02T21:16:35,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/54aa845c8da943a48df6e4e467922e83 is 50, key is test_row_0/B:col10/1733174195300/Put/seqid=0 2024-12-02T21:16:35,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742115_1291 (size=12301) 2024-12-02T21:16:35,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174255959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:36,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/54aa845c8da943a48df6e4e467922e83 2024-12-02T21:16:36,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/bdbaa607af35495f9dfe7e46c9860c4e is 50, key is test_row_0/C:col10/1733174195300/Put/seqid=0 2024-12-02T21:16:36,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742116_1292 (size=12301) 2024-12-02T21:16:36,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174256368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:36,370 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:36,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174256468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:36,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/bdbaa607af35495f9dfe7e46c9860c4e 2024-12-02T21:16:36,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/6c117199b95e4ac9bc527819f1763288 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6c117199b95e4ac9bc527819f1763288 2024-12-02T21:16:36,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6c117199b95e4ac9bc527819f1763288, entries=200, sequenceid=290, filesize=14.4 K 2024-12-02T21:16:36,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/54aa845c8da943a48df6e4e467922e83 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/54aa845c8da943a48df6e4e467922e83 2024-12-02T21:16:36,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/54aa845c8da943a48df6e4e467922e83, entries=150, sequenceid=290, filesize=12.0 K 2024-12-02T21:16:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/bdbaa607af35495f9dfe7e46c9860c4e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/bdbaa607af35495f9dfe7e46c9860c4e 2024-12-02T21:16:36,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/bdbaa607af35495f9dfe7e46c9860c4e, entries=150, sequenceid=290, filesize=12.0 K 2024-12-02T21:16:36,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 744ddead248cd8e4726f3c3f3c9ab799 in 1250ms, sequenceid=290, compaction requested=true 2024-12-02T21:16:36,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:36,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:36,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:36,551 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:36,551 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:36,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:36,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:36,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:36,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:36,552 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:36,552 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:36,552 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/A is initiating minor compaction (all files) 2024-12-02T21:16:36,552 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/B is initiating minor compaction (all files) 2024-12-02T21:16:36,552 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/B in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:36,552 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/A in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:36,552 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81649006a72049c888706089980756c8, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/b10140cf4de54582b37fbb745aca25f7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/54aa845c8da943a48df6e4e467922e83] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=36.2 K 2024-12-02T21:16:36,552 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3344faf7e2e447fe97ed59135369743c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/e02ee56497da4aba86dac4f0a0a2d40b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6c117199b95e4ac9bc527819f1763288] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=38.6 K 2024-12-02T21:16:36,552 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 81649006a72049c888706089980756c8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733174191430 2024-12-02T21:16:36,553 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3344faf7e2e447fe97ed59135369743c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733174191430 2024-12-02T21:16:36,553 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b10140cf4de54582b37fbb745aca25f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733174192576 2024-12-02T21:16:36,553 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e02ee56497da4aba86dac4f0a0a2d40b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733174192576 2024-12-02T21:16:36,553 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 54aa845c8da943a48df6e4e467922e83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733174194205 2024-12-02T21:16:36,553 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c117199b95e4ac9bc527819f1763288, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733174194205 2024-12-02T21:16:36,559 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:36,559 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:36,560 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/d1b9a0b4e9e34f7da5e68117ded2fd4c is 50, key is test_row_0/A:col10/1733174195300/Put/seqid=0 2024-12-02T21:16:36,560 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/cfd91f07e87e41b092210bbd74150409 is 50, key is test_row_0/B:col10/1733174195300/Put/seqid=0 2024-12-02T21:16:36,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742117_1293 (size=12915) 2024-12-02T21:16:36,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742118_1294 (size=12915) 2024-12-02T21:16:36,582 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/d1b9a0b4e9e34f7da5e68117ded2fd4c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/d1b9a0b4e9e34f7da5e68117ded2fd4c 2024-12-02T21:16:36,587 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/A of 744ddead248cd8e4726f3c3f3c9ab799 into d1b9a0b4e9e34f7da5e68117ded2fd4c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:36,587 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:36,587 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/A, priority=13, startTime=1733174196551; duration=0sec 2024-12-02T21:16:36,587 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:36,587 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:A 2024-12-02T21:16:36,587 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:36,588 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:36,588 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/C is initiating minor compaction (all files) 2024-12-02T21:16:36,588 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/C in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:36,588 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/cbce339d72994251a29f3e320543db05, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ca1c9e4a7fa34caa98b170c39a2dac92, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/bdbaa607af35495f9dfe7e46c9860c4e] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=36.2 K 2024-12-02T21:16:36,589 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbce339d72994251a29f3e320543db05, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733174191430 2024-12-02T21:16:36,589 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca1c9e4a7fa34caa98b170c39a2dac92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733174192576 2024-12-02T21:16:36,589 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdbaa607af35495f9dfe7e46c9860c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733174194205 2024-12-02T21:16:36,595 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:36,595 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/321d07cb86de44e8949a2d383b93ee3c is 50, key is test_row_0/C:col10/1733174195300/Put/seqid=0 2024-12-02T21:16:36,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742119_1295 (size=12915) 2024-12-02T21:16:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T21:16:36,609 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:36,609 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-02T21:16:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T21:16:36,611 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:36,612 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:36,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T21:16:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:36,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:16:36,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:36,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:36,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:36,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:36,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:36,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:36,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/f85178f7b59445ac9d8446a6533eaf06 is 50, key is test_row_0/A:col10/1733174196750/Put/seqid=0 2024-12-02T21:16:36,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742120_1296 (size=9857) 2024-12-02T21:16:36,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:36,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:36,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174196753, max timestamp=1733174196778Num uniques 859; 2024-12-02T21:16:36,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174196753, max timestamp=1733174196778Num uniques 859; 2024-12-02T21:16:36,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:16:36,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:16:36,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174196775, max timestamp=1733174196778Num uniques 859; 2024-12-02T21:16:36,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:36,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174256799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:36,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174256800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:36,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174256801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:36,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:36,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174256908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174256908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:36,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174256908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:36,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:36,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T21:16:36,968 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/cfd91f07e87e41b092210bbd74150409 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cfd91f07e87e41b092210bbd74150409 2024-12-02T21:16:36,973 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/B of 744ddead248cd8e4726f3c3f3c9ab799 into cfd91f07e87e41b092210bbd74150409(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:36,973 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:36,973 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/B, priority=13, startTime=1733174196551; duration=0sec 2024-12-02T21:16:36,973 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:36,973 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:B 2024-12-02T21:16:37,004 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/321d07cb86de44e8949a2d383b93ee3c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/321d07cb86de44e8949a2d383b93ee3c 2024-12-02T21:16:37,008 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/C of 744ddead248cd8e4726f3c3f3c9ab799 into 321d07cb86de44e8949a2d383b93ee3c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:37,008 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:37,008 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/C, priority=13, startTime=1733174196551; duration=0sec 2024-12-02T21:16:37,008 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:37,008 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:C 2024-12-02T21:16:37,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:37,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174257119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174257119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174257119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/f85178f7b59445ac9d8446a6533eaf06 2024-12-02T21:16:37,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/81d42609d5b84851947fb47d885203fb is 50, key is test_row_0/B:col10/1733174196750/Put/seqid=0 2024-12-02T21:16:37,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742121_1297 (size=9857) 2024-12-02T21:16:37,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:37,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:37,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T21:16:37,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:37,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:37,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174257429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174257429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174257429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174257478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,480 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/81d42609d5b84851947fb47d885203fb 2024-12-02T21:16:37,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/33411368abd840b3bcc0fe35aecc4a69 is 50, key is test_row_0/C:col10/1733174196750/Put/seqid=0 2024-12-02T21:16:37,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742122_1298 (size=9857) 2024-12-02T21:16:37,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T21:16:37,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:37,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:37,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174257939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174257939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174257939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:37,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:37,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:37,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/33411368abd840b3bcc0fe35aecc4a69 2024-12-02T21:16:37,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/f85178f7b59445ac9d8446a6533eaf06 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f85178f7b59445ac9d8446a6533eaf06 2024-12-02T21:16:37,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f85178f7b59445ac9d8446a6533eaf06, entries=100, sequenceid=302, filesize=9.6 K 2024-12-02T21:16:37,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/81d42609d5b84851947fb47d885203fb as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81d42609d5b84851947fb47d885203fb 2024-12-02T21:16:38,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81d42609d5b84851947fb47d885203fb, entries=100, sequenceid=302, filesize=9.6 K 2024-12-02T21:16:38,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/33411368abd840b3bcc0fe35aecc4a69 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/33411368abd840b3bcc0fe35aecc4a69 2024-12-02T21:16:38,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/33411368abd840b3bcc0fe35aecc4a69, entries=100, sequenceid=302, filesize=9.6 K 2024-12-02T21:16:38,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 744ddead248cd8e4726f3c3f3c9ab799 in 1256ms, sequenceid=302, compaction requested=false 2024-12-02T21:16:38,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:38,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T21:16:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:38,138 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-12-02T21:16:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:38,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/85bf61d92fdd43e4afed21614ff18cf7 is 50, key is test_row_0/A:col10/1733174196794/Put/seqid=0 2024-12-02T21:16:38,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742123_1299 (size=12301) 2024-12-02T21:16:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:38,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:38,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174258404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:38,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174258518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:38,547 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/85bf61d92fdd43e4afed21614ff18cf7 2024-12-02T21:16:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/9e00cffb5f7c476bb4cfbfbf8e04e8e6 is 50, key is test_row_0/B:col10/1733174196794/Put/seqid=0 2024-12-02T21:16:38,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742124_1300 (size=12301) 2024-12-02T21:16:38,558 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/9e00cffb5f7c476bb4cfbfbf8e04e8e6 2024-12-02T21:16:38,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/4e4e2d0d002e455cb190d2c5b2d5b7af is 50, key is test_row_0/C:col10/1733174196794/Put/seqid=0 2024-12-02T21:16:38,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742125_1301 (size=12301) 2024-12-02T21:16:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174258728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T21:16:38,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174258948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:38,950 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174258959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174258959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:38,961 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:38,961 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in 744ddead248cd8e4726f3c3f3c9ab799 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:38,969 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/4e4e2d0d002e455cb190d2c5b2d5b7af 2024-12-02T21:16:39,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174259038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:39,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/85bf61d92fdd43e4afed21614ff18cf7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/85bf61d92fdd43e4afed21614ff18cf7 2024-12-02T21:16:39,045 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/85bf61d92fdd43e4afed21614ff18cf7, entries=150, sequenceid=339, filesize=12.0 K 2024-12-02T21:16:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/9e00cffb5f7c476bb4cfbfbf8e04e8e6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/9e00cffb5f7c476bb4cfbfbf8e04e8e6 2024-12-02T21:16:39,049 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/9e00cffb5f7c476bb4cfbfbf8e04e8e6, entries=150, sequenceid=339, filesize=12.0 K 2024-12-02T21:16:39,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/4e4e2d0d002e455cb190d2c5b2d5b7af as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4e4e2d0d002e455cb190d2c5b2d5b7af 2024-12-02T21:16:39,053 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4e4e2d0d002e455cb190d2c5b2d5b7af, entries=150, sequenceid=339, filesize=12.0 K 2024-12-02T21:16:39,054 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=53.67 KB/54960 for 744ddead248cd8e4726f3c3f3c9ab799 in 916ms, sequenceid=339, compaction requested=true 2024-12-02T21:16:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-02T21:16:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-02T21:16:39,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-02T21:16:39,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4430 sec 2024-12-02T21:16:39,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.4480 sec 2024-12-02T21:16:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:39,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T21:16:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:39,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/2a87bfaecee14842bffb96426aee3ce7 is 50, key is test_row_0/A:col10/1733174198396/Put/seqid=0 2024-12-02T21:16:39,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742126_1302 (size=14741) 2024-12-02T21:16:39,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/2a87bfaecee14842bffb96426aee3ce7 2024-12-02T21:16:39,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/5f05222ea5d64b7fb08fde949ef94fb1 is 50, key is test_row_0/B:col10/1733174198396/Put/seqid=0 2024-12-02T21:16:39,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742127_1303 (size=12301) 2024-12-02T21:16:39,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/5f05222ea5d64b7fb08fde949ef94fb1 2024-12-02T21:16:39,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/4d9fdd2d2dbd44dc946054a9c715678a is 50, key is test_row_0/C:col10/1733174198396/Put/seqid=0 2024-12-02T21:16:39,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742128_1304 (size=12301) 2024-12-02T21:16:39,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174259564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:39,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174259564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:39,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174259678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:39,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174259678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:39,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174259889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:39,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174259889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:39,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/4d9fdd2d2dbd44dc946054a9c715678a 2024-12-02T21:16:39,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/2a87bfaecee14842bffb96426aee3ce7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2a87bfaecee14842bffb96426aee3ce7 2024-12-02T21:16:39,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2a87bfaecee14842bffb96426aee3ce7, entries=200, sequenceid=351, filesize=14.4 K 2024-12-02T21:16:39,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/5f05222ea5d64b7fb08fde949ef94fb1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/5f05222ea5d64b7fb08fde949ef94fb1 2024-12-02T21:16:39,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/5f05222ea5d64b7fb08fde949ef94fb1, entries=150, sequenceid=351, filesize=12.0 K 2024-12-02T21:16:39,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/4d9fdd2d2dbd44dc946054a9c715678a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4d9fdd2d2dbd44dc946054a9c715678a 2024-12-02T21:16:39,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4d9fdd2d2dbd44dc946054a9c715678a, entries=150, sequenceid=351, filesize=12.0 K 2024-12-02T21:16:39,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 744ddead248cd8e4726f3c3f3c9ab799 in 462ms, sequenceid=351, compaction requested=true 2024-12-02T21:16:39,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:39,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:39,973 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:16:39,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:39,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:39,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:39,973 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:16:39,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744ddead248cd8e4726f3c3f3c9ab799:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:39,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:39,974 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:16:39,975 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/B is initiating minor compaction (all files) 2024-12-02T21:16:39,975 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/B in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:39,975 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cfd91f07e87e41b092210bbd74150409, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81d42609d5b84851947fb47d885203fb, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/9e00cffb5f7c476bb4cfbfbf8e04e8e6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/5f05222ea5d64b7fb08fde949ef94fb1] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=46.3 K 2024-12-02T21:16:39,975 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49814 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:16:39,975 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/A is initiating minor compaction (all files) 2024-12-02T21:16:39,975 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/A in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:39,975 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/d1b9a0b4e9e34f7da5e68117ded2fd4c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f85178f7b59445ac9d8446a6533eaf06, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/85bf61d92fdd43e4afed21614ff18cf7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2a87bfaecee14842bffb96426aee3ce7] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=48.6 K 2024-12-02T21:16:39,975 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting cfd91f07e87e41b092210bbd74150409, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733174194205 2024-12-02T21:16:39,975 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b9a0b4e9e34f7da5e68117ded2fd4c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733174194205 2024-12-02T21:16:39,976 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d42609d5b84851947fb47d885203fb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733174195327 2024-12-02T21:16:39,976 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f85178f7b59445ac9d8446a6533eaf06, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733174195327 2024-12-02T21:16:39,976 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85bf61d92fdd43e4afed21614ff18cf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733174196794 2024-12-02T21:16:39,976 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e00cffb5f7c476bb4cfbfbf8e04e8e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733174196794 2024-12-02T21:16:39,976 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a87bfaecee14842bffb96426aee3ce7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733174198396 2024-12-02T21:16:39,976 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f05222ea5d64b7fb08fde949ef94fb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733174198396 2024-12-02T21:16:39,985 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#B#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:39,985 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#A#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:39,985 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/4c8376e8454646f08bcb31095378bea9 is 50, key is test_row_0/A:col10/1733174198396/Put/seqid=0 2024-12-02T21:16:39,985 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/6b27cd642cdc44e1b6f3a5f98b2d0209 is 50, key is test_row_0/B:col10/1733174198396/Put/seqid=0 2024-12-02T21:16:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742130_1306 (size=13051) 2024-12-02T21:16:39,995 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/4c8376e8454646f08bcb31095378bea9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/4c8376e8454646f08bcb31095378bea9 2024-12-02T21:16:39,999 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/A of 744ddead248cd8e4726f3c3f3c9ab799 into 4c8376e8454646f08bcb31095378bea9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:39,999 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:39,999 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/A, priority=12, startTime=1733174199973; duration=0sec 2024-12-02T21:16:39,999 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:39,999 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:A 2024-12-02T21:16:39,999 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:16:39,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742129_1305 (size=13051) 2024-12-02T21:16:40,001 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:16:40,001 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 744ddead248cd8e4726f3c3f3c9ab799/C is initiating minor compaction (all files) 2024-12-02T21:16:40,001 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744ddead248cd8e4726f3c3f3c9ab799/C in TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:40,001 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/321d07cb86de44e8949a2d383b93ee3c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/33411368abd840b3bcc0fe35aecc4a69, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4e4e2d0d002e455cb190d2c5b2d5b7af, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4d9fdd2d2dbd44dc946054a9c715678a] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp, totalSize=46.3 K 2024-12-02T21:16:40,001 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 321d07cb86de44e8949a2d383b93ee3c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733174194205 2024-12-02T21:16:40,002 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33411368abd840b3bcc0fe35aecc4a69, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733174195327 2024-12-02T21:16:40,002 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4e2d0d002e455cb190d2c5b2d5b7af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733174196794 2024-12-02T21:16:40,002 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d9fdd2d2dbd44dc946054a9c715678a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733174198396 2024-12-02T21:16:40,010 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744ddead248cd8e4726f3c3f3c9ab799#C#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:40,011 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ac51a89a64c34bb7bf2288b82173cd7e is 50, key is test_row_0/C:col10/1733174198396/Put/seqid=0 2024-12-02T21:16:40,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742131_1307 (size=13051) 2024-12-02T21:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:40,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T21:16:40,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174199556, max timestamp=1733174200201Num uniques 859; 2024-12-02T21:16:40,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174199556, max timestamp=1733174200201Num uniques 859; 2024-12-02T21:16:40,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:40,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:40,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174199556, max timestamp=1733174200201Num uniques 859; 2024-12-02T21:16:40,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:40,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:40,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:40,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:40,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:40,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:40,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:40,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/cf0b0323a288469bb84b56dfc519c4f1 is 50, key is test_row_0/A:col10/1733174200201/Put/seqid=0 2024-12-02T21:16:40,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742132_1308 (size=14741) 2024-12-02T21:16:40,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174200236, max timestamp=1733174200246Num uniques 859; 2024-12-02T21:16:40,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:40,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174200236, max timestamp=1733174200246Num uniques 859; 2024-12-02T21:16:40,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:40,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174200236, max timestamp=1733174200246Num uniques 859; 2024-12-02T21:16:40,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:40,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733174200269, max timestamp=1733174200290Num uniques 1018; 2024-12-02T21:16:40,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733174200269, max timestamp=1733174200290Num uniques 1018; 2024-12-02T21:16:40,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T21:16:40,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-02T21:16:40,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733174200269, max timestamp=1733174200290Num uniques 1018; 2024-12-02T21:16:40,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T21:16:40,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733174200322, max timestamp=1733174200338Num uniques 1077; 2024-12-02T21:16:40,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-02T21:16:40,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733174200322, max timestamp=1733174200338Num uniques 1027; 2024-12-02T21:16:40,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-02T21:16:40,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733174200322, max timestamp=1733174200338Num uniques 1027; 2024-12-02T21:16:40,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-02T21:16:40,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733174200362, max timestamp=1733174200384Num uniques 1086; 2024-12-02T21:16:40,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8513, totalHeapSize=19584, min timestamp=1733174200362, max timestamp=1733174200384Num uniques 1086; 2024-12-02T21:16:40,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49733, new segment data size=8513, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-02T21:16:40,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=8506, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-02T21:16:40,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8510, totalHeapSize=19584, min timestamp=1733174200362, max timestamp=1733174200384Num uniques 1086; 2024-12-02T21:16:40,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49730, new segment data size=8510, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-02T21:16:40,405 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/6b27cd642cdc44e1b6f3a5f98b2d0209 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/6b27cd642cdc44e1b6f3a5f98b2d0209 2024-12-02T21:16:40,410 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/B of 744ddead248cd8e4726f3c3f3c9ab799 into 6b27cd642cdc44e1b6f3a5f98b2d0209(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:40,410 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:40,410 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/B, priority=12, startTime=1733174199973; duration=0sec 2024-12-02T21:16:40,410 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:40,410 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:B 2024-12-02T21:16:40,422 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/ac51a89a64c34bb7bf2288b82173cd7e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ac51a89a64c34bb7bf2288b82173cd7e 2024-12-02T21:16:40,426 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744ddead248cd8e4726f3c3f3c9ab799/C of 744ddead248cd8e4726f3c3f3c9ab799 into ac51a89a64c34bb7bf2288b82173cd7e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:40,426 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:40,426 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., storeName=744ddead248cd8e4726f3c3f3c9ab799/C, priority=12, startTime=1733174199973; duration=0sec 2024-12-02T21:16:40,426 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:40,426 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744ddead248cd8e4726f3c3f3c9ab799:C 2024-12-02T21:16:40,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733174200428, max timestamp=1733174200444Num uniques 1045; 2024-12-02T21:16:40,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733174200428, max timestamp=1733174200444Num uniques 1045; 2024-12-02T21:16:40,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-02T21:16:40,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-02T21:16:40,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733174200428, max timestamp=1733174200444Num uniques 1045; 2024-12-02T21:16:40,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-02T21:16:40,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733174200473, max timestamp=1733174200499Num uniques 1054; 2024-12-02T21:16:40,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-02T21:16:40,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733174200473, max timestamp=1733174200499Num uniques 1054; 2024-12-02T21:16:40,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-02T21:16:40,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733174200473, max timestamp=1733174200499Num uniques 1054; 2024-12-02T21:16:40,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-02T21:16:40,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733174200512, max timestamp=1733174200531Num uniques 1013; 2024-12-02T21:16:40,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-02T21:16:40,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733174200512, max timestamp=1733174200531Num uniques 1013; 2024-12-02T21:16:40,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-02T21:16:40,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733174200512, max timestamp=1733174200531Num uniques 1013; 2024-12-02T21:16:40,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-02T21:16:40,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733174200573, max timestamp=1733174200582Num uniques 1022; 2024-12-02T21:16:40,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-02T21:16:40,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733174200573, max timestamp=1733174200582Num uniques 1022; 2024-12-02T21:16:40,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-02T21:16:40,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733174200573, max timestamp=1733174200582Num uniques 1022; 2024-12-02T21:16:40,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-02T21:16:40,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/cf0b0323a288469bb84b56dfc519c4f1 2024-12-02T21:16:40,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733174200607, max timestamp=1733174200616Num uniques 1031; 2024-12-02T21:16:40,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-02T21:16:40,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733174200607, max timestamp=1733174200616Num uniques 1031; 2024-12-02T21:16:40,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-02T21:16:40,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733174200607, max timestamp=1733174200616Num uniques 1031; 2024-12-02T21:16:40,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-02T21:16:40,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/07c92190511f4ee2b0af0f2aee85da03 is 50, key is test_row_0/B:col10/1733174200201/Put/seqid=0 2024-12-02T21:16:40,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742133_1309 (size=12301) 2024-12-02T21:16:40,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733174200651, max timestamp=1733174200663Num uniques 1090; 2024-12-02T21:16:40,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-02T21:16:40,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733174200651, max timestamp=1733174200663Num uniques 1090; 2024-12-02T21:16:40,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-02T21:16:40,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733174200651, max timestamp=1733174200663Num uniques 1090; 2024-12-02T21:16:40,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-02T21:16:40,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733174200695, max timestamp=1733174200712Num uniques 1049; 2024-12-02T21:16:40,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-02T21:16:40,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733174200695, max timestamp=1733174200712Num uniques 1049; 2024-12-02T21:16:40,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733174200695, max timestamp=1733174200712Num uniques 1049; 2024-12-02T21:16:40,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-02T21:16:40,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-02T21:16:40,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733174200758, max timestamp=1733174200765Num uniques 1058; 2024-12-02T21:16:40,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-02T21:16:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733174200758, max timestamp=1733174200765Num uniques 1058; 2024-12-02T21:16:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-02T21:16:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733174200758, max timestamp=1733174200765Num uniques 1058; 2024-12-02T21:16:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-02T21:16:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T21:16:40,769 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:40,770 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-02T21:16:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T21:16:40,772 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:40,772 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:40,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:40,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733174200789, max timestamp=1733174200805Num uniques 1017; 2024-12-02T21:16:40,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733174200789, max timestamp=1733174200805Num uniques 1017; 2024-12-02T21:16:40,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-02T21:16:40,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733174200789, max timestamp=1733174200805Num uniques 1017; 2024-12-02T21:16:40,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-02T21:16:40,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-02T21:16:40,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733174200848, max timestamp=1733174200858Num uniques 1026; 2024-12-02T21:16:40,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-02T21:16:40,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733174200848, max timestamp=1733174200858Num uniques 1026; 2024-12-02T21:16:40,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-02T21:16:40,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733174200848, max timestamp=1733174200858Num uniques 1026; 2024-12-02T21:16:40,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-02T21:16:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T21:16:40,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733174200882, max timestamp=1733174200898Num uniques 1035; 2024-12-02T21:16:40,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-02T21:16:40,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733174200892, max timestamp=1733174200898Num uniques 1035; 2024-12-02T21:16:40,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-12-02T21:16:40,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733174200892, max timestamp=1733174200898Num uniques 1035; 2024-12-02T21:16:40,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-12-02T21:16:40,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T21:16:40,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:40,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:40,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:40,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:40,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:40,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1733174260981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:40,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733174200952, max timestamp=1733174200985Num uniques 1094; 2024-12-02T21:16:40,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-12-02T21:16:40,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733174200952, max timestamp=1733174200985Num uniques 1044; 2024-12-02T21:16:40,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733174200952, max timestamp=1733174200985Num uniques 1044; 2024-12-02T21:16:40,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-12-02T21:16:40,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-12-02T21:16:41,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733174201002, max timestamp=1733174201010Num uniques 1103; 2024-12-02T21:16:41,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-12-02T21:16:41,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733174201002, max timestamp=1733174201010Num uniques 1103; 2024-12-02T21:16:41,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-12-02T21:16:41,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733174201002, max timestamp=1733174201010Num uniques 1103; 2024-12-02T21:16:41,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-12-02T21:16:41,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/07c92190511f4ee2b0af0f2aee85da03 2024-12-02T21:16:41,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/0adb6cd22cb4494bbb6459bcb495fdad is 50, key is test_row_0/C:col10/1733174200201/Put/seqid=0 2024-12-02T21:16:41,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742134_1310 (size=12301) 2024-12-02T21:16:41,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:41,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174261034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174261035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:41,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733174201032, max timestamp=1733174201039Num uniques 1062; 2024-12-02T21:16:41,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-12-02T21:16:41,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733174201032, max timestamp=1733174201039Num uniques 1062; 2024-12-02T21:16:41,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-12-02T21:16:41,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733174201032, max timestamp=1733174201039Num uniques 1062; 2024-12-02T21:16:41,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-12-02T21:16:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:41,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1733174201065, max timestamp=1733174201074Num uniques 1021; 2024-12-02T21:16:41,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-12-02T21:16:41,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1733174201065, max timestamp=1733174201074Num uniques 1021; 2024-12-02T21:16:41,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1733174201065, max timestamp=1733174201074Num uniques 1021; 2024-12-02T21:16:41,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-12-02T21:16:41,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-12-02T21:16:41,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T21:16:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:41,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T21:16:41,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1733174201098, max timestamp=1733174201106Num uniques 1080; 2024-12-02T21:16:41,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1733174201098, max timestamp=1733174201106Num uniques 1080; 2024-12-02T21:16:41,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-12-02T21:16:41,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-12-02T21:16:41,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1733174201098, max timestamp=1733174201106Num uniques 1080; 2024-12-02T21:16:41,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-12-02T21:16:41,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1733174201113, max timestamp=1733174201133Num uniques 1089; 2024-12-02T21:16:41,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-12-02T21:16:41,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1733174201113, max timestamp=1733174201133Num uniques 1089; 2024-12-02T21:16:41,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-12-02T21:16:41,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1733174201113, max timestamp=1733174201133Num uniques 1089; 2024-12-02T21:16:41,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-12-02T21:16:41,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1733174201152, max timestamp=1733174201158Num uniques 1148; 2024-12-02T21:16:41,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=11358, suffix heap size=147360,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=248 2024-12-02T21:16:41,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1733174201152, max timestamp=1733174201158Num uniques 1148; 2024-12-02T21:16:41,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=11358, suffix heap size=147360,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=248 2024-12-02T21:16:41,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1733174201152, max timestamp=1733174201158Num uniques 1148; 2024-12-02T21:16:41,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=11358, suffix heap size=147360,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=248 2024-12-02T21:16:41,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53044 deadline: 1733174261180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:41,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53010 deadline: 1733174261179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:41,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=257, cellSize=11762, totalHeapSize=26968, min timestamp=1733174201170, max timestamp=1733174201185Num uniques 1107; 2024-12-02T21:16:41,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50692, new segment data size=11762, suffix heap size=143096,new segment heap size=26968 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=257 2024-12-02T21:16:41,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=179, cellSize=8194, totalHeapSize=18856, min timestamp=1733174201170, max timestamp=1733174201185Num uniques 1107; 2024-12-02T21:16:41,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50692, new segment data size=8194, suffix heap size=143096,new segment heap size=18856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=179 2024-12-02T21:16:41,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=257, cellSize=11762, totalHeapSize=26968, min timestamp=1733174201170, max timestamp=1733174201185Num uniques 1107; 2024-12-02T21:16:41,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50692, new segment data size=11762, suffix heap size=143096,new segment heap size=26968 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=257 2024-12-02T21:16:41,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T21:16:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:41,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:41,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53084 deadline: 1733174261253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53068 deadline: 1733174261254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:41,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733174201219, max timestamp=1733174201255Num uniques 1116; 2024-12-02T21:16:41,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51106, new segment data size=9886, suffix heap size=144032,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1116, new segment cells count=216 2024-12-02T21:16:41,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733174201219, max timestamp=1733174201255Num uniques 1116; 2024-12-02T21:16:41,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51106, new segment data size=9886, suffix heap size=144032,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1116, new segment cells count=216 2024-12-02T21:16:41,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733174201219, max timestamp=1733174201255Num uniques 1038; 2024-12-02T21:16:41,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47538, new segment data size=9886, suffix heap size=135920,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1038, new segment cells count=216 2024-12-02T21:16:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:16:41,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=275, cellSize=12590, totalHeapSize=28840, min timestamp=1733174201293, max timestamp=1733174201310Num uniques 1075; 2024-12-02T21:16:41,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10300, totalHeapSize=23640, min timestamp=1733174201293, max timestamp=1733174201310Num uniques 1075; 2024-12-02T21:16:41,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10300, totalHeapSize=23640, min timestamp=1733174201293, max timestamp=1733174201310Num uniques 1075; 2024-12-02T21:16:41,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=10300, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-12-02T21:16:41,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=12590, suffix heap size=139768,new segment heap size=28840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=275 2024-12-02T21:16:41,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=10300, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-12-02T21:16:41,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8414, totalHeapSize=19376, min timestamp=1733174201323, max timestamp=1733174201343Num uniques 1134; 2024-12-02T21:16:41,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8414, totalHeapSize=19376, min timestamp=1733174201323, max timestamp=1733174201343Num uniques 1084; 2024-12-02T21:16:41,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51924, new segment data size=8414, suffix heap size=145904,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1134, new segment cells count=184 2024-12-02T21:16:41,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49634, new segment data size=8414, suffix heap size=140704,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1084, new segment cells count=184 2024-12-02T21:16:41,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8414, totalHeapSize=19376, min timestamp=1733174201323, max timestamp=1733174201343Num uniques 1084; 2024-12-02T21:16:41,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49634, new segment data size=8414, suffix heap size=140704,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1084, new segment cells count=184 2024-12-02T21:16:41,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=193, cellSize=8838, totalHeapSize=20312, min timestamp=1733174201364, max timestamp=1733174201370Num uniques 1043; 2024-12-02T21:16:41,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47768, new segment data size=8838, suffix heap size=136440,new segment heap size=20312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1043, new segment cells count=193 2024-12-02T21:16:41,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733174201364, max timestamp=1733174201370Num uniques 1043; 2024-12-02T21:16:41,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47768, new segment data size=6870, suffix heap size=136440,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1043, new segment cells count=150 2024-12-02T21:16:41,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733174201364, max timestamp=1733174201370Num uniques 1043; 2024-12-02T21:16:41,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47768, new segment data size=6870, suffix heap size=136440,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1043, new segment cells count=150 2024-12-02T21:16:41,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T21:16:41,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:41,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. as already flushing 2024-12-02T21:16:41,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:41,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:41,385 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-12-02T21:16:41,385 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,385 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,385 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,386 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-12-02T21:16:41,386 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-12-02T21:16:41,386 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,386 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,386 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,386 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,386 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,386 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,387 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-12-02T21:16:41,387 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,387 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,387 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,387 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-12-02T21:16:41,387 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,387 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,387 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,387 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,387 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,388 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-12-02T21:16:41,388 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,388 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,388 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,388 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,389 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-12-02T21:16:41,389 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,389 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,389 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,390 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-12-02T21:16:41,390 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,390 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,390 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,392 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-12-02T21:16:41,392 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,392 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,392 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,393 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-12-02T21:16:41,393 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:16:41,393 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,393 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:41,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T21:16:41,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/0adb6cd22cb4494bbb6459bcb495fdad 2024-12-02T21:16:41,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/cf0b0323a288469bb84b56dfc519c4f1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/cf0b0323a288469bb84b56dfc519c4f1 2024-12-02T21:16:41,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/cf0b0323a288469bb84b56dfc519c4f1, entries=200, sequenceid=374, filesize=14.4 K 2024-12-02T21:16:41,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/07c92190511f4ee2b0af0f2aee85da03 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/07c92190511f4ee2b0af0f2aee85da03 2024-12-02T21:16:41,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/07c92190511f4ee2b0af0f2aee85da03, entries=150, sequenceid=374, filesize=12.0 K 2024-12-02T21:16:41,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/0adb6cd22cb4494bbb6459bcb495fdad as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0adb6cd22cb4494bbb6459bcb495fdad 2024-12-02T21:16:41,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0adb6cd22cb4494bbb6459bcb495fdad, entries=150, sequenceid=374, filesize=12.0 K 2024-12-02T21:16:41,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=123.63 KB/126594 for 744ddead248cd8e4726f3c3f3c9ab799 in 1263ms, sequenceid=374, compaction requested=false 2024-12-02T21:16:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:41,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T21:16:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:41,537 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 744ddead248cd8e4726f3c3f3c9ab799 3/3 column families, dataSize=123.63 KB heapSize=353.14 KB 2024-12-02T21:16:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=A 2024-12-02T21:16:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=B 2024-12-02T21:16:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744ddead248cd8e4726f3c3f3c9ab799, store=C 2024-12-02T21:16:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/33350d3ac7f146bea787e18821fa7bda is 50, key is test_row_0/A:col10/1733174201387/Put/seqid=0 2024-12-02T21:16:41,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742135_1311 (size=12301) 2024-12-02T21:16:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T21:16:41,946 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=858 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/33350d3ac7f146bea787e18821fa7bda 2024-12-02T21:16:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/7231e1c5b9e646ddbbf6a38508d173cd is 50, key is test_row_0/B:col10/1733174201387/Put/seqid=0 2024-12-02T21:16:41,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742136_1312 (size=12301) 2024-12-02T21:16:42,179 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T21:16:42,364 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.57 KB at sequenceid=858 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/7231e1c5b9e646ddbbf6a38508d173cd 2024-12-02T21:16:42,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/48257b9c75954c81af0200fca665168c is 50, key is test_row_0/C:col10/1733174201387/Put/seqid=0 2024-12-02T21:16:42,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742137_1313 (size=12301) 2024-12-02T21:16:42,780 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.57 KB at sequenceid=858 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/48257b9c75954c81af0200fca665168c 2024-12-02T21:16:42,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/A/33350d3ac7f146bea787e18821fa7bda as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/33350d3ac7f146bea787e18821fa7bda 2024-12-02T21:16:42,792 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/33350d3ac7f146bea787e18821fa7bda, entries=150, sequenceid=858, filesize=12.0 K 2024-12-02T21:16:42,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/B/7231e1c5b9e646ddbbf6a38508d173cd as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7231e1c5b9e646ddbbf6a38508d173cd 2024-12-02T21:16:42,797 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7231e1c5b9e646ddbbf6a38508d173cd, entries=150, sequenceid=858, filesize=12.0 K 2024-12-02T21:16:42,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/.tmp/C/48257b9c75954c81af0200fca665168c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/48257b9c75954c81af0200fca665168c 2024-12-02T21:16:42,802 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/48257b9c75954c81af0200fca665168c, entries=150, sequenceid=858, filesize=12.0 K 2024-12-02T21:16:42,803 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~123.63 KB/126594, heapSize ~353.09 KB/361568, currentSize=0 B/0 for 744ddead248cd8e4726f3c3f3c9ab799 in 1266ms, sequenceid=858, compaction requested=true 2024-12-02T21:16:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 744ddead248cd8e4726f3c3f3c9ab799: 2024-12-02T21:16:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-02T21:16:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-02T21:16:42,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-02T21:16:42,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0310 sec 2024-12-02T21:16:42,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.0370 sec 2024-12-02T21:16:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T21:16:42,920 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 272 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 243 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4609 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4647 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4418 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4600 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4642 2024-12-02T21:16:42,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T21:16:42,921 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:16:42,921 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:16:42,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:42,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:42,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:16:42,923 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T21:16:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T21:16:42,928 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174202927"}]},"ts":"1733174202927"} 2024-12-02T21:16:42,929 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T21:16:42,929 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T21:16:42,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T21:16:42,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744ddead248cd8e4726f3c3f3c9ab799, UNASSIGN}] 2024-12-02T21:16:42,932 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744ddead248cd8e4726f3c3f3c9ab799, UNASSIGN 2024-12-02T21:16:42,933 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=744ddead248cd8e4726f3c3f3c9ab799, regionState=CLOSING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:42,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=744ddead248cd8e4726f3c3f3c9ab799, UNASSIGN because future has completed 2024-12-02T21:16:42,935 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:16:42,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:16:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T21:16:43,088 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:43,088 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:16:43,089 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 744ddead248cd8e4726f3c3f3c9ab799, disabling compactions & flushes 2024-12-02T21:16:43,089 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:43,089 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:43,089 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. after waiting 0 ms 2024-12-02T21:16:43,089 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:43,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fc5e145bd3be495f981035883b90980a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/60cbe57e71ba4e22a3e41cc1334eeb7d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/54ded5732f3b4ce98e1b84b79dbf0fce, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/195675cc92a543488cdda16a48072af9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/5eb31409b613454b841968b85b203767, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/a58e382f3e684f6484e54e6e7dde32e2, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2e5660c278494ecb8e3e4bb7b901a2a5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f240a59f007e494094f6c96aad432d2d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/162b7c015a574ed498afa9ad719812e6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/4f6d10b3b3494405a57bdc4b45bc7c05, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fedfb46a303b4111b250f676d0c9253a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6d21af1a159545adafdc32a253a1bc6a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/594bee5428494d9a8fb379f832441227, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/9abbfb2bcc3141ee8243c4aff437cf34, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3344faf7e2e447fe97ed59135369743c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3099e6ecce844f8fb310b597bbc79fc2, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/e02ee56497da4aba86dac4f0a0a2d40b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6c117199b95e4ac9bc527819f1763288, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/d1b9a0b4e9e34f7da5e68117ded2fd4c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f85178f7b59445ac9d8446a6533eaf06, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/85bf61d92fdd43e4afed21614ff18cf7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2a87bfaecee14842bffb96426aee3ce7] to archive 2024-12-02T21:16:43,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:16:43,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fc5e145bd3be495f981035883b90980a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fc5e145bd3be495f981035883b90980a 2024-12-02T21:16:43,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/60cbe57e71ba4e22a3e41cc1334eeb7d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/60cbe57e71ba4e22a3e41cc1334eeb7d 2024-12-02T21:16:43,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/54ded5732f3b4ce98e1b84b79dbf0fce to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/54ded5732f3b4ce98e1b84b79dbf0fce 2024-12-02T21:16:43,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/195675cc92a543488cdda16a48072af9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/195675cc92a543488cdda16a48072af9 2024-12-02T21:16:43,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/5eb31409b613454b841968b85b203767 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/5eb31409b613454b841968b85b203767 2024-12-02T21:16:43,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/a58e382f3e684f6484e54e6e7dde32e2 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/a58e382f3e684f6484e54e6e7dde32e2 2024-12-02T21:16:43,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2e5660c278494ecb8e3e4bb7b901a2a5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2e5660c278494ecb8e3e4bb7b901a2a5 2024-12-02T21:16:43,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f240a59f007e494094f6c96aad432d2d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f240a59f007e494094f6c96aad432d2d 2024-12-02T21:16:43,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/162b7c015a574ed498afa9ad719812e6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/162b7c015a574ed498afa9ad719812e6 2024-12-02T21:16:43,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/4f6d10b3b3494405a57bdc4b45bc7c05 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/4f6d10b3b3494405a57bdc4b45bc7c05 2024-12-02T21:16:43,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fedfb46a303b4111b250f676d0c9253a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/fedfb46a303b4111b250f676d0c9253a 2024-12-02T21:16:43,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6d21af1a159545adafdc32a253a1bc6a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6d21af1a159545adafdc32a253a1bc6a 2024-12-02T21:16:43,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/594bee5428494d9a8fb379f832441227 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/594bee5428494d9a8fb379f832441227 2024-12-02T21:16:43,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/9abbfb2bcc3141ee8243c4aff437cf34 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/9abbfb2bcc3141ee8243c4aff437cf34 2024-12-02T21:16:43,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3344faf7e2e447fe97ed59135369743c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3344faf7e2e447fe97ed59135369743c 2024-12-02T21:16:43,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3099e6ecce844f8fb310b597bbc79fc2 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/3099e6ecce844f8fb310b597bbc79fc2 2024-12-02T21:16:43,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/e02ee56497da4aba86dac4f0a0a2d40b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/e02ee56497da4aba86dac4f0a0a2d40b 2024-12-02T21:16:43,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6c117199b95e4ac9bc527819f1763288 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/6c117199b95e4ac9bc527819f1763288 2024-12-02T21:16:43,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/d1b9a0b4e9e34f7da5e68117ded2fd4c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/d1b9a0b4e9e34f7da5e68117ded2fd4c 2024-12-02T21:16:43,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f85178f7b59445ac9d8446a6533eaf06 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/f85178f7b59445ac9d8446a6533eaf06 2024-12-02T21:16:43,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/85bf61d92fdd43e4afed21614ff18cf7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/85bf61d92fdd43e4afed21614ff18cf7 2024-12-02T21:16:43,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2a87bfaecee14842bffb96426aee3ce7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/2a87bfaecee14842bffb96426aee3ce7 2024-12-02T21:16:43,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/4fb6f4c0e7da46dabfa2c7daf670a8be, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e52c742dfeb94f54821c89c8715225e9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/77601444eb1244159bd5e67261b86c36, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/0dc8fa2fd02840019583fed09a2f8fa5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/33208cb6de334c14855723f44e4b2797, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e335074ab04b43faa39ad910fb1b1269, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7be3499b979b40dbb34c9cdc27459dc4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7d4555d53df94d29b80c7fad2a131331, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/f988f55d84d34c92bb641cc6b80bacc6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cf6e05152201440f87c78be532322764, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cef03b1634294454b691bec345dfb718, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/d7e7b2a6dd724a289ef3323c026a4bee, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/6cd46e9b1b2846f5837af0a7be582331, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/73174f9da2f34d87a15731d8299772e0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81649006a72049c888706089980756c8, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/72cead9101a449859ae2120e3bcec0a1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/b10140cf4de54582b37fbb745aca25f7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cfd91f07e87e41b092210bbd74150409, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/54aa845c8da943a48df6e4e467922e83, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81d42609d5b84851947fb47d885203fb, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/9e00cffb5f7c476bb4cfbfbf8e04e8e6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/5f05222ea5d64b7fb08fde949ef94fb1] to archive 2024-12-02T21:16:43,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:16:43,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/4fb6f4c0e7da46dabfa2c7daf670a8be to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/4fb6f4c0e7da46dabfa2c7daf670a8be 2024-12-02T21:16:43,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e52c742dfeb94f54821c89c8715225e9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e52c742dfeb94f54821c89c8715225e9 2024-12-02T21:16:43,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/77601444eb1244159bd5e67261b86c36 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/77601444eb1244159bd5e67261b86c36 2024-12-02T21:16:43,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/0dc8fa2fd02840019583fed09a2f8fa5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/0dc8fa2fd02840019583fed09a2f8fa5 2024-12-02T21:16:43,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/33208cb6de334c14855723f44e4b2797 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/33208cb6de334c14855723f44e4b2797 2024-12-02T21:16:43,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e335074ab04b43faa39ad910fb1b1269 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/e335074ab04b43faa39ad910fb1b1269 2024-12-02T21:16:43,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7be3499b979b40dbb34c9cdc27459dc4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7be3499b979b40dbb34c9cdc27459dc4 2024-12-02T21:16:43,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7d4555d53df94d29b80c7fad2a131331 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7d4555d53df94d29b80c7fad2a131331 2024-12-02T21:16:43,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/f988f55d84d34c92bb641cc6b80bacc6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/f988f55d84d34c92bb641cc6b80bacc6 2024-12-02T21:16:43,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cf6e05152201440f87c78be532322764 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cf6e05152201440f87c78be532322764 2024-12-02T21:16:43,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cef03b1634294454b691bec345dfb718 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cef03b1634294454b691bec345dfb718 2024-12-02T21:16:43,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/d7e7b2a6dd724a289ef3323c026a4bee to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/d7e7b2a6dd724a289ef3323c026a4bee 2024-12-02T21:16:43,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/6cd46e9b1b2846f5837af0a7be582331 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/6cd46e9b1b2846f5837af0a7be582331 2024-12-02T21:16:43,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/73174f9da2f34d87a15731d8299772e0 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/73174f9da2f34d87a15731d8299772e0 2024-12-02T21:16:43,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81649006a72049c888706089980756c8 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81649006a72049c888706089980756c8 2024-12-02T21:16:43,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/72cead9101a449859ae2120e3bcec0a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/72cead9101a449859ae2120e3bcec0a1 2024-12-02T21:16:43,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/b10140cf4de54582b37fbb745aca25f7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/b10140cf4de54582b37fbb745aca25f7 2024-12-02T21:16:43,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cfd91f07e87e41b092210bbd74150409 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/cfd91f07e87e41b092210bbd74150409 2024-12-02T21:16:43,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/54aa845c8da943a48df6e4e467922e83 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/54aa845c8da943a48df6e4e467922e83 2024-12-02T21:16:43,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81d42609d5b84851947fb47d885203fb to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/81d42609d5b84851947fb47d885203fb 2024-12-02T21:16:43,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/9e00cffb5f7c476bb4cfbfbf8e04e8e6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/9e00cffb5f7c476bb4cfbfbf8e04e8e6 2024-12-02T21:16:43,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/5f05222ea5d64b7fb08fde949ef94fb1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/5f05222ea5d64b7fb08fde949ef94fb1 2024-12-02T21:16:43,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e54f4844223f43f9bb1bbcbaee97fef7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f4d7ddd4955f41ebb65e635abf7a9331, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ee8b60517d174e4c952aa0f321f99e5d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4252b9725d404aa195e4e196040b7917, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/3741ee2c171f4b91b7171d63b6f9841e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e19cf643f2fe4254836f0d39fe0bc8a1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f8af32901a02454cb9812f267a817f94, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0ba8f534d9524995b976d0dfce567956, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/80732d60c67e4740968c56c225ae2342, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/aa099a30b1c14f31a7a4fdbea086b044, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/039d887d18bf4d0281c7a296cffa163a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/64aa4d640094474087aeff21bd8e9790, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f55e1a9ecc244932ae55b1704db3d353, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/24ad8f773c444393817dcddbc471a5dc, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/cbce339d72994251a29f3e320543db05, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ed827d6a28d142908b042c4b4073d5f8, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ca1c9e4a7fa34caa98b170c39a2dac92, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/321d07cb86de44e8949a2d383b93ee3c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/bdbaa607af35495f9dfe7e46c9860c4e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/33411368abd840b3bcc0fe35aecc4a69, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4e4e2d0d002e455cb190d2c5b2d5b7af, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4d9fdd2d2dbd44dc946054a9c715678a] to archive 2024-12-02T21:16:43,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:16:43,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e54f4844223f43f9bb1bbcbaee97fef7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e54f4844223f43f9bb1bbcbaee97fef7 2024-12-02T21:16:43,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f4d7ddd4955f41ebb65e635abf7a9331 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f4d7ddd4955f41ebb65e635abf7a9331 2024-12-02T21:16:43,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ee8b60517d174e4c952aa0f321f99e5d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ee8b60517d174e4c952aa0f321f99e5d 2024-12-02T21:16:43,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4252b9725d404aa195e4e196040b7917 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4252b9725d404aa195e4e196040b7917 2024-12-02T21:16:43,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/3741ee2c171f4b91b7171d63b6f9841e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/3741ee2c171f4b91b7171d63b6f9841e 2024-12-02T21:16:43,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e19cf643f2fe4254836f0d39fe0bc8a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/e19cf643f2fe4254836f0d39fe0bc8a1 2024-12-02T21:16:43,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f8af32901a02454cb9812f267a817f94 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f8af32901a02454cb9812f267a817f94 2024-12-02T21:16:43,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0ba8f534d9524995b976d0dfce567956 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0ba8f534d9524995b976d0dfce567956 2024-12-02T21:16:43,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/80732d60c67e4740968c56c225ae2342 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/80732d60c67e4740968c56c225ae2342 2024-12-02T21:16:43,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/aa099a30b1c14f31a7a4fdbea086b044 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/aa099a30b1c14f31a7a4fdbea086b044 2024-12-02T21:16:43,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/039d887d18bf4d0281c7a296cffa163a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/039d887d18bf4d0281c7a296cffa163a 2024-12-02T21:16:43,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/64aa4d640094474087aeff21bd8e9790 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/64aa4d640094474087aeff21bd8e9790 2024-12-02T21:16:43,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f55e1a9ecc244932ae55b1704db3d353 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/f55e1a9ecc244932ae55b1704db3d353 2024-12-02T21:16:43,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/24ad8f773c444393817dcddbc471a5dc to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/24ad8f773c444393817dcddbc471a5dc 2024-12-02T21:16:43,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/cbce339d72994251a29f3e320543db05 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/cbce339d72994251a29f3e320543db05 2024-12-02T21:16:43,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ed827d6a28d142908b042c4b4073d5f8 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ed827d6a28d142908b042c4b4073d5f8 2024-12-02T21:16:43,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ca1c9e4a7fa34caa98b170c39a2dac92 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ca1c9e4a7fa34caa98b170c39a2dac92 2024-12-02T21:16:43,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/321d07cb86de44e8949a2d383b93ee3c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/321d07cb86de44e8949a2d383b93ee3c 2024-12-02T21:16:43,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/bdbaa607af35495f9dfe7e46c9860c4e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/bdbaa607af35495f9dfe7e46c9860c4e 2024-12-02T21:16:43,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/33411368abd840b3bcc0fe35aecc4a69 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/33411368abd840b3bcc0fe35aecc4a69 2024-12-02T21:16:43,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4e4e2d0d002e455cb190d2c5b2d5b7af to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4e4e2d0d002e455cb190d2c5b2d5b7af 2024-12-02T21:16:43,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4d9fdd2d2dbd44dc946054a9c715678a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/4d9fdd2d2dbd44dc946054a9c715678a 2024-12-02T21:16:43,164 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/recovered.edits/861.seqid, newMaxSeqId=861, maxSeqId=1 2024-12-02T21:16:43,165 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799. 2024-12-02T21:16:43,165 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 744ddead248cd8e4726f3c3f3c9ab799: Waiting for close lock at 1733174203089Running coprocessor pre-close hooks at 1733174203089Disabling compacts and flushes for region at 1733174203089Disabling writes for close at 1733174203089Writing region close event to WAL at 1733174203162 (+73 ms)Running coprocessor post-close hooks at 1733174203165 (+3 ms)Closed at 1733174203165 2024-12-02T21:16:43,166 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:43,167 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=744ddead248cd8e4726f3c3f3c9ab799, regionState=CLOSED 2024-12-02T21:16:43,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:16:43,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-02T21:16:43,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 744ddead248cd8e4726f3c3f3c9ab799, server=87c3fdb6c570,35563,1733174113859 in 234 msec 2024-12-02T21:16:43,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-02T21:16:43,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744ddead248cd8e4726f3c3f3c9ab799, UNASSIGN in 239 msec 2024-12-02T21:16:43,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-02T21:16:43,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 243 msec 2024-12-02T21:16:43,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174203175"}]},"ts":"1733174203175"} 2024-12-02T21:16:43,177 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T21:16:43,177 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T21:16:43,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 254 msec 2024-12-02T21:16:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T21:16:43,240 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:43,241 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T21:16:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:43,244 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T21:16:43,246 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:43,250 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:43,254 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/recovered.edits] 2024-12-02T21:16:43,260 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/33350d3ac7f146bea787e18821fa7bda to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/33350d3ac7f146bea787e18821fa7bda 2024-12-02T21:16:43,261 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/4c8376e8454646f08bcb31095378bea9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/4c8376e8454646f08bcb31095378bea9 2024-12-02T21:16:43,262 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/cf0b0323a288469bb84b56dfc519c4f1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/A/cf0b0323a288469bb84b56dfc519c4f1 2024-12-02T21:16:43,264 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/07c92190511f4ee2b0af0f2aee85da03 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/07c92190511f4ee2b0af0f2aee85da03 2024-12-02T21:16:43,264 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/6b27cd642cdc44e1b6f3a5f98b2d0209 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/6b27cd642cdc44e1b6f3a5f98b2d0209 2024-12-02T21:16:43,265 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7231e1c5b9e646ddbbf6a38508d173cd to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/B/7231e1c5b9e646ddbbf6a38508d173cd 2024-12-02T21:16:43,267 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0adb6cd22cb4494bbb6459bcb495fdad to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/0adb6cd22cb4494bbb6459bcb495fdad 2024-12-02T21:16:43,267 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/48257b9c75954c81af0200fca665168c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/48257b9c75954c81af0200fca665168c 2024-12-02T21:16:43,268 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ac51a89a64c34bb7bf2288b82173cd7e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/C/ac51a89a64c34bb7bf2288b82173cd7e 2024-12-02T21:16:43,270 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/recovered.edits/861.seqid to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799/recovered.edits/861.seqid 2024-12-02T21:16:43,270 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/744ddead248cd8e4726f3c3f3c9ab799 2024-12-02T21:16:43,270 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T21:16:43,272 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:43,274 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T21:16:43,276 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T21:16:43,277 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:43,277 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T21:16:43,277 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733174203277"}]},"ts":"9223372036854775807"} 2024-12-02T21:16:43,278 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T21:16:43,278 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 744ddead248cd8e4726f3c3f3c9ab799, NAME => 'TestAcidGuarantees,,1733174179168.744ddead248cd8e4726f3c3f3c9ab799.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T21:16:43,278 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T21:16:43,279 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733174203278"}]},"ts":"9223372036854775807"} 2024-12-02T21:16:43,280 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T21:16:43,280 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:43,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-12-02T21:16:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T21:16:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T21:16:43,350 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:43,358 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T21:16:43,364 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1368367939_22 at /127.0.0.1:36198 [Waiting for operation #344] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1636387191_22 at /127.0.0.1:54404 [Waiting for operation #132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=1903 (was 1952) 2024-12-02T21:16:43,374 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=1902 2024-12-02T21:16:43,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T21:16:43,376 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:16:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:43,378 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T21:16:43,378 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:43,379 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-12-02T21:16:43,379 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T21:16:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-02T21:16:43,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742138_1314 (size=1061) 2024-12-02T21:16:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-02T21:16:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-02T21:16:43,790 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0 2024-12-02T21:16:43,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742139_1315 (size=53) 2024-12-02T21:16:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-02T21:16:44,202 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:16:44,202 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a3e315252bf6ea46b1e790171c7a96a1, disabling compactions & flushes 2024-12-02T21:16:44,202 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:44,202 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:44,203 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. after waiting 0 ms 2024-12-02T21:16:44,203 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:44,203 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:44,203 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a3e315252bf6ea46b1e790171c7a96a1: Waiting for close lock at 1733174204202Disabling compacts and flushes for region at 1733174204202Disabling writes for close at 1733174204203 (+1 ms)Writing region close event to WAL at 1733174204203Closed at 1733174204203 2024-12-02T21:16:44,272 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T21:16:44,273 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733174204273"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174204273"}]},"ts":"1733174204273"} 2024-12-02T21:16:44,278 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T21:16:44,279 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T21:16:44,279 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174204279"}]},"ts":"1733174204279"} 2024-12-02T21:16:44,281 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T21:16:44,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, ASSIGN}] 2024-12-02T21:16:44,283 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, ASSIGN 2024-12-02T21:16:44,284 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, ASSIGN; state=OFFLINE, location=87c3fdb6c570,35563,1733174113859; forceNewPlan=false, retain=false 2024-12-02T21:16:44,435 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a3e315252bf6ea46b1e790171c7a96a1, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:44,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, ASSIGN because future has completed 2024-12-02T21:16:44,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:16:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-02T21:16:44,598 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:44,599 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:16:44,599 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,599 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:16:44,599 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,599 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,601 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,602 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:16:44,602 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e315252bf6ea46b1e790171c7a96a1 columnFamilyName A 2024-12-02T21:16:44,602 DEBUG [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:44,603 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(327): Store=a3e315252bf6ea46b1e790171c7a96a1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:16:44,603 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,604 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:16:44,604 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e315252bf6ea46b1e790171c7a96a1 columnFamilyName B 2024-12-02T21:16:44,604 DEBUG [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:44,605 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(327): Store=a3e315252bf6ea46b1e790171c7a96a1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:16:44,605 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,606 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:16:44,606 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e315252bf6ea46b1e790171c7a96a1 columnFamilyName C 2024-12-02T21:16:44,606 DEBUG [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:44,606 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(327): Store=a3e315252bf6ea46b1e790171c7a96a1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:16:44,607 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:44,607 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,607 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,608 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,609 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,609 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,609 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:16:44,611 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,613 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:16:44,613 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened a3e315252bf6ea46b1e790171c7a96a1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74631753, jitterRate=0.1120997816324234}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:16:44,613 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:44,614 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for a3e315252bf6ea46b1e790171c7a96a1: Running coprocessor pre-open hook at 1733174204599Writing region info on filesystem at 1733174204599Initializing all the Stores at 1733174204600 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174204600Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174204600Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174204600Cleaning up temporary data from old regions at 1733174204609 (+9 ms)Running coprocessor post-open hooks at 1733174204613 (+4 ms)Region opened successfully at 1733174204614 (+1 ms) 2024-12-02T21:16:44,615 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., pid=89, masterSystemTime=1733174204592 2024-12-02T21:16:44,617 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:44,617 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:44,618 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a3e315252bf6ea46b1e790171c7a96a1, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:44,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:16:44,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-02T21:16:44,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 in 182 msec 2024-12-02T21:16:44,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-02T21:16:44,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, ASSIGN in 340 msec 2024-12-02T21:16:44,623 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T21:16:44,624 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174204624"}]},"ts":"1733174204624"} 2024-12-02T21:16:44,625 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T21:16:44,626 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T21:16:44,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2500 sec 2024-12-02T21:16:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-02T21:16:45,539 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:45,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:45,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:45,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:45,541 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a2e459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:45,543 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108d24d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:45,544 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:45,546 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:45,546 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T21:16:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 87c3fdb6c570,39577,1733174113031 2024-12-02T21:16:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f57bb87 2024-12-02T21:16:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T21:16:45,548 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T21:16:45,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T21:16:45,550 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:16:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T21:16:45,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742140_1316 (size=1097) 2024-12-02T21:16:45,967 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T21:16:45,967 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T21:16:45,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T21:16:45,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, REOPEN/MOVE}] 2024-12-02T21:16:45,977 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, REOPEN/MOVE 2024-12-02T21:16:45,978 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a3e315252bf6ea46b1e790171c7a96a1, regionState=CLOSING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:45,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, REOPEN/MOVE because future has completed 2024-12-02T21:16:45,980 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:16:45,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:16:46,134 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,134 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:16:46,134 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing a3e315252bf6ea46b1e790171c7a96a1, disabling compactions & flushes 2024-12-02T21:16:46,134 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,134 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,134 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. after waiting 0 ms 2024-12-02T21:16:46,134 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,141 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:16:46,142 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,142 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for a3e315252bf6ea46b1e790171c7a96a1: Waiting for close lock at 1733174206134Running coprocessor pre-close hooks at 1733174206134Disabling compacts and flushes for region at 1733174206134Disabling writes for close at 1733174206134Writing region close event to WAL at 1733174206135 (+1 ms)Running coprocessor post-close hooks at 1733174206142 (+7 ms)Closed at 1733174206142 2024-12-02T21:16:46,142 WARN [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: a3e315252bf6ea46b1e790171c7a96a1 to self. 2024-12-02T21:16:46,145 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,145 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a3e315252bf6ea46b1e790171c7a96a1, regionState=CLOSED 2024-12-02T21:16:46,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:16:46,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-02T21:16:46,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 in 168 msec 2024-12-02T21:16:46,152 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, REOPEN/MOVE; state=CLOSED, location=87c3fdb6c570,35563,1733174113859; forceNewPlan=false, retain=true 2024-12-02T21:16:46,303 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a3e315252bf6ea46b1e790171c7a96a1, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, REOPEN/MOVE because future has completed 2024-12-02T21:16:46,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:16:46,467 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,467 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:16:46,468 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,468 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:16:46,468 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,468 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,469 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,470 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:16:46,471 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e315252bf6ea46b1e790171c7a96a1 columnFamilyName A 2024-12-02T21:16:46,472 DEBUG [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:46,473 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(327): Store=a3e315252bf6ea46b1e790171c7a96a1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:16:46,473 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,474 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:16:46,474 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e315252bf6ea46b1e790171c7a96a1 columnFamilyName B 2024-12-02T21:16:46,474 DEBUG [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:46,475 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(327): Store=a3e315252bf6ea46b1e790171c7a96a1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:16:46,475 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,475 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:16:46,476 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e315252bf6ea46b1e790171c7a96a1 columnFamilyName C 2024-12-02T21:16:46,476 DEBUG [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:46,476 INFO [StoreOpener-a3e315252bf6ea46b1e790171c7a96a1-1 {}] regionserver.HStore(327): Store=a3e315252bf6ea46b1e790171c7a96a1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:16:46,476 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,476 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,477 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,478 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,478 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,478 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,479 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:16:46,480 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,480 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened a3e315252bf6ea46b1e790171c7a96a1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60142872, jitterRate=-0.10380136966705322}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:16:46,480 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,481 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for a3e315252bf6ea46b1e790171c7a96a1: Running coprocessor pre-open hook at 1733174206468Writing region info on filesystem at 1733174206468Initializing all the Stores at 1733174206469 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174206469Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174206469Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174206469Cleaning up temporary data from old regions at 1733174206478 (+9 ms)Running coprocessor post-open hooks at 1733174206480 (+2 ms)Region opened successfully at 1733174206481 (+1 ms) 2024-12-02T21:16:46,481 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., pid=94, masterSystemTime=1733174206462 2024-12-02T21:16:46,482 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,483 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,483 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a3e315252bf6ea46b1e790171c7a96a1, regionState=OPEN, openSeqNum=5, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:16:46,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-12-02T21:16:46,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 in 177 msec 2024-12-02T21:16:46,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-02T21:16:46,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, REOPEN/MOVE in 510 msec 2024-12-02T21:16:46,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-02T21:16:46,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-12-02T21:16:46,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-12-02T21:16:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T21:16:46,492 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:46,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,494 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c477195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,496 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa1ac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,498 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c01f88b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,499 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542a404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,500 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a85ddb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,501 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1958a999, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,503 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a40437a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,504 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6a8b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,505 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707d9108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,506 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2d50de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,508 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc9603e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,509 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15eeed8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,510 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3516609b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,511 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59348db5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,513 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a9ca8e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,514 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@daed07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,516 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd0b945, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626892cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:16:46,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:16:46,520 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:16:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:16:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:16:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7773c449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:16:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:16:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:16:46,521 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:16:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84b69b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:16:46,522 DEBUG [Thread-1451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,522 DEBUG [Thread-1470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,522 DEBUG [Thread-1455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,522 DEBUG [Thread-1464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,523 DEBUG [Thread-1459 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,523 DEBUG [Thread-1468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,523 DEBUG [Thread-1466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,524 DEBUG [Thread-1455 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,524 DEBUG [Thread-1462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,524 DEBUG [Thread-1455 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,525 DEBUG [Thread-1453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,526 DEBUG [Thread-1457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:16:46,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-02T21:16:46,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:16:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T21:16:46,528 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,531 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:46,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:16:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:16:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:46,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025c73ea40a6eb48ceb62f340ecaf7bf12_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174206535/Put/seqid=0 2024-12-02T21:16:46,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174206565, max timestamp=1733174206580Num uniques 859; 2024-12-02T21:16:46,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174206565, max timestamp=1733174206580Num uniques 859; 2024-12-02T21:16:46,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:16:46,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:16:46,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733174206565, max timestamp=1733174206580Num uniques 859; 2024-12-02T21:16:46,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:16:46,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742141_1317 (size=12154) 2024-12-02T21:16:46,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174266602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174266607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174266609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174266609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60834 deadline: 1733174266610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T21:16:46,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:46,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174266719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174266728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174266728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174266729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60834 deadline: 1733174266729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T21:16:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174266939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174266939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174266939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60834 deadline: 1733174266940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174266940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:46,987 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:46,990 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025c73ea40a6eb48ceb62f340ecaf7bf12_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025c73ea40a6eb48ceb62f340ecaf7bf12_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:46,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/6f92526a1b344447938c558e80d8b01a, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:46,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/6f92526a1b344447938c558e80d8b01a is 175, key is test_row_0/A:col10/1733174206535/Put/seqid=0 2024-12-02T21:16:46,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:46,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:46,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:46,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:46,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:46,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742142_1318 (size=30955) 2024-12-02T21:16:47,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:47,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:47,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T21:16:47,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174267249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174267259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174267259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174267259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60834 deadline: 1733174267259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,398 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/6f92526a1b344447938c558e80d8b01a 2024-12-02T21:16:47,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/8d1046b39e2f4edfa1476071af1d8757 is 50, key is test_row_0/B:col10/1733174206535/Put/seqid=0 2024-12-02T21:16:47,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742143_1319 (size=12001) 2024-12-02T21:16:47,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T21:16:47,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:47,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:47,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174267768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174267768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174267768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174267769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60834 deadline: 1733174267779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:47,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:47,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/8d1046b39e2f4edfa1476071af1d8757 2024-12-02T21:16:47,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/110f671141d94643bdaac2de3f3bea93 is 50, key is test_row_0/C:col10/1733174206535/Put/seqid=0 2024-12-02T21:16:47,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742144_1320 (size=12001) 2024-12-02T21:16:47,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:47,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:47,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:47,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:48,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:48,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:48,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:48,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:48,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:48,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:48,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/110f671141d94643bdaac2de3f3bea93 2024-12-02T21:16:48,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/6f92526a1b344447938c558e80d8b01a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/6f92526a1b344447938c558e80d8b01a 2024-12-02T21:16:48,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/6f92526a1b344447938c558e80d8b01a, entries=150, sequenceid=14, filesize=30.2 K 2024-12-02T21:16:48,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/8d1046b39e2f4edfa1476071af1d8757 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/8d1046b39e2f4edfa1476071af1d8757 2024-12-02T21:16:48,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/8d1046b39e2f4edfa1476071af1d8757, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T21:16:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/110f671141d94643bdaac2de3f3bea93 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/110f671141d94643bdaac2de3f3bea93 2024-12-02T21:16:48,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/110f671141d94643bdaac2de3f3bea93, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T21:16:48,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a3e315252bf6ea46b1e790171c7a96a1 in 1744ms, sequenceid=14, compaction requested=false 2024-12-02T21:16:48,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:48,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T21:16:48,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:48,370 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-02T21:16:48,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:48,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:48,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:48,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027ea83612b6ee42058b355268bde8ecf8_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174206608/Put/seqid=0 2024-12-02T21:16:48,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742145_1321 (size=12154) 2024-12-02T21:16:48,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:48,386 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027ea83612b6ee42058b355268bde8ecf8_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027ea83612b6ee42058b355268bde8ecf8_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d is 175, key is test_row_0/A:col10/1733174206608/Put/seqid=0 2024-12-02T21:16:48,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742146_1322 (size=30955) 2024-12-02T21:16:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T21:16:48,690 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T21:16:48,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:48,793 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d 2024-12-02T21:16:48,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174268790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:48,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174268791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:48,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/f4082892ac0c46b58188854dff262bcd is 50, key is test_row_0/B:col10/1733174206608/Put/seqid=0 2024-12-02T21:16:48,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174268792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:48,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174268793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:48,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742147_1323 (size=12001) 2024-12-02T21:16:48,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60834 deadline: 1733174268798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:48,806 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:48,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174268899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:48,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174268908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174268908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:48,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174268908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:49,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174269109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:49,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:49,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174269118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:49,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174269118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174269118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:49,204 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/f4082892ac0c46b58188854dff262bcd 2024-12-02T21:16:49,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/a698d1d30bf74e01b9fc39440ff0db49 is 50, key is test_row_0/C:col10/1733174206608/Put/seqid=0 2024-12-02T21:16:49,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742148_1324 (size=12001) 2024-12-02T21:16:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174269419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174269429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174269429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174269429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:49,665 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/a698d1d30bf74e01b9fc39440ff0db49 2024-12-02T21:16:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d 2024-12-02T21:16:49,672 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d, entries=150, sequenceid=49, filesize=30.2 K 2024-12-02T21:16:49,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/f4082892ac0c46b58188854dff262bcd as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/f4082892ac0c46b58188854dff262bcd 2024-12-02T21:16:49,675 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/f4082892ac0c46b58188854dff262bcd, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T21:16:49,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/a698d1d30bf74e01b9fc39440ff0db49 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/a698d1d30bf74e01b9fc39440ff0db49 2024-12-02T21:16:49,679 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/a698d1d30bf74e01b9fc39440ff0db49, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T21:16:49,680 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for a3e315252bf6ea46b1e790171c7a96a1 in 1310ms, sequenceid=49, compaction requested=false 2024-12-02T21:16:49,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:49,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:49,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-02T21:16:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-02T21:16:49,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-02T21:16:49,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1500 sec 2024-12-02T21:16:49,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.1600 sec 2024-12-02T21:16:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:49,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:16:49,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:49,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:49,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:49,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:49,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:49,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:49,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023cd5bf9527634cfaba63986397a20592_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174209941/Put/seqid=0 2024-12-02T21:16:49,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742149_1325 (size=14594) 2024-12-02T21:16:49,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174209976, max timestamp=1733174209988Num uniques 859; 2024-12-02T21:16:49,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174209976, max timestamp=1733174209988Num uniques 859; 2024-12-02T21:16:49,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174209976, max timestamp=1733174209988Num uniques 859; 2024-12-02T21:16:49,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:49,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:49,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:49,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174269981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174270039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174270040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174270040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174270099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174270158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174270158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174270159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174270319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:50,365 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023cd5bf9527634cfaba63986397a20592_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023cd5bf9527634cfaba63986397a20592_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:50,366 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/448f4f63c4194a35893b92c1eac199ef, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:50,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/448f4f63c4194a35893b92c1eac199ef is 175, key is test_row_0/A:col10/1733174209941/Put/seqid=0 2024-12-02T21:16:50,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742150_1326 (size=39549) 2024-12-02T21:16:50,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174270369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174270369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174270369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174270629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174270678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174270678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174270679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T21:16:50,699 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:50,700 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-02T21:16:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T21:16:50,701 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:50,702 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:50,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:50,770 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/448f4f63c4194a35893b92c1eac199ef 2024-12-02T21:16:50,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/9ebdbfc68da44eedaec4b2fdb3535f35 is 50, key is test_row_0/B:col10/1733174209941/Put/seqid=0 2024-12-02T21:16:50,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742151_1327 (size=12001) 2024-12-02T21:16:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T21:16:50,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60834 deadline: 1733174270828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:50,832 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:50,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-02T21:16:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:50,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-02T21:16:51,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:51,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T21:16:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174271139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:51,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-02T21:16:51,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:51,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/9ebdbfc68da44eedaec4b2fdb3535f35 2024-12-02T21:16:51,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/86dee15f75144f5584b6017901c2fdf8 is 50, key is test_row_0/C:col10/1733174209941/Put/seqid=0 2024-12-02T21:16:51,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742152_1328 (size=12001) 2024-12-02T21:16:51,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174271198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174271198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174271198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:51,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-02T21:16:51,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:51,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T21:16:51,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-02T21:16:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:51,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/86dee15f75144f5584b6017901c2fdf8 2024-12-02T21:16:51,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/448f4f63c4194a35893b92c1eac199ef as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/448f4f63c4194a35893b92c1eac199ef 2024-12-02T21:16:51,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/448f4f63c4194a35893b92c1eac199ef, entries=200, sequenceid=60, filesize=38.6 K 2024-12-02T21:16:51,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/9ebdbfc68da44eedaec4b2fdb3535f35 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9ebdbfc68da44eedaec4b2fdb3535f35 2024-12-02T21:16:51,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9ebdbfc68da44eedaec4b2fdb3535f35, entries=150, sequenceid=60, filesize=11.7 K 2024-12-02T21:16:51,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/86dee15f75144f5584b6017901c2fdf8 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/86dee15f75144f5584b6017901c2fdf8 2024-12-02T21:16:51,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/86dee15f75144f5584b6017901c2fdf8, entries=150, sequenceid=60, filesize=11.7 K 2024-12-02T21:16:51,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a3e315252bf6ea46b1e790171c7a96a1 in 1668ms, sequenceid=60, compaction requested=true 2024-12-02T21:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:51,610 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:51,610 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:51,611 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:51,611 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:51,611 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/B is initiating minor compaction (all files) 2024-12-02T21:16:51,611 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/A is initiating minor compaction (all files) 2024-12-02T21:16:51,611 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/B in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,611 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/A in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,611 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/8d1046b39e2f4edfa1476071af1d8757, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/f4082892ac0c46b58188854dff262bcd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9ebdbfc68da44eedaec4b2fdb3535f35] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=35.2 K 2024-12-02T21:16:51,611 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/6f92526a1b344447938c558e80d8b01a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/448f4f63c4194a35893b92c1eac199ef] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=99.1 K 2024-12-02T21:16:51,611 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,611 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/6f92526a1b344447938c558e80d8b01a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/448f4f63c4194a35893b92c1eac199ef] 2024-12-02T21:16:51,611 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d1046b39e2f4edfa1476071af1d8757, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733174206535 2024-12-02T21:16:51,611 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f92526a1b344447938c558e80d8b01a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733174206535 2024-12-02T21:16:51,612 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f4082892ac0c46b58188854dff262bcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733174206606 2024-12-02T21:16:51,612 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d2c15eaf06e4ba1b6e6b06d6eb5556d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733174206606 2024-12-02T21:16:51,612 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ebdbfc68da44eedaec4b2fdb3535f35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733174208787 2024-12-02T21:16:51,612 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 448f4f63c4194a35893b92c1eac199ef, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733174208787 2024-12-02T21:16:51,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-02T21:16:51,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:51,616 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T21:16:51,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:51,618 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:51,619 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202a2e45fcdd89b42a3a16e18f782964e39_a3e315252bf6ea46b1e790171c7a96a1 store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:51,621 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#B#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:51,621 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202a2e45fcdd89b42a3a16e18f782964e39_a3e315252bf6ea46b1e790171c7a96a1, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:51,621 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/a46dd1d942c64afa8642dec4d554fc8b is 50, key is test_row_0/B:col10/1733174209941/Put/seqid=0 2024-12-02T21:16:51,621 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a2e45fcdd89b42a3a16e18f782964e39_a3e315252bf6ea46b1e790171c7a96a1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:51,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ed89441a183e42a4b91a7e3f08d23dcc_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174210038/Put/seqid=0 2024-12-02T21:16:51,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742153_1329 (size=12104) 2024-12-02T21:16:51,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742154_1330 (size=4469) 2024-12-02T21:16:51,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742155_1331 (size=12154) 2024-12-02T21:16:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T21:16:52,051 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#A#compaction#281 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:52,051 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/dbbf5ef162aa4e0b8a766969c22f52fd is 175, key is test_row_0/A:col10/1733174209941/Put/seqid=0 2024-12-02T21:16:52,053 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/a46dd1d942c64afa8642dec4d554fc8b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/a46dd1d942c64afa8642dec4d554fc8b 2024-12-02T21:16:52,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742156_1332 (size=31058) 2024-12-02T21:16:52,058 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/B of a3e315252bf6ea46b1e790171c7a96a1 into a46dd1d942c64afa8642dec4d554fc8b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:52,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:52,058 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:52,058 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/B, priority=13, startTime=1733174211610; duration=0sec 2024-12-02T21:16:52,058 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:52,058 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:B 2024-12-02T21:16:52,058 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:52,059 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:52,059 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/C is initiating minor compaction (all files) 2024-12-02T21:16:52,059 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/C in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:52,059 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/110f671141d94643bdaac2de3f3bea93, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/a698d1d30bf74e01b9fc39440ff0db49, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/86dee15f75144f5584b6017901c2fdf8] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=35.2 K 2024-12-02T21:16:52,060 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 110f671141d94643bdaac2de3f3bea93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733174206535 2024-12-02T21:16:52,061 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting a698d1d30bf74e01b9fc39440ff0db49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733174206606 2024-12-02T21:16:52,061 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 86dee15f75144f5584b6017901c2fdf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733174208787 2024-12-02T21:16:52,062 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ed89441a183e42a4b91a7e3f08d23dcc_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ed89441a183e42a4b91a7e3f08d23dcc_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:52,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/9ecb234774e94c04b6e5250f1e268cc4, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:52,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/9ecb234774e94c04b6e5250f1e268cc4 is 175, key is test_row_0/A:col10/1733174210038/Put/seqid=0 2024-12-02T21:16:52,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742157_1333 (size=30955) 2024-12-02T21:16:52,068 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/9ecb234774e94c04b6e5250f1e268cc4 2024-12-02T21:16:52,074 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:52,075 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/20f52c84ddb6440e9216d64eb866aef9 is 50, key is test_row_0/C:col10/1733174209941/Put/seqid=0 2024-12-02T21:16:52,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/4fcda08de5cb44c68c015369fcde2a7a is 50, key is test_row_0/B:col10/1733174210038/Put/seqid=0 2024-12-02T21:16:52,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742158_1334 (size=12104) 2024-12-02T21:16:52,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742159_1335 (size=12001) 2024-12-02T21:16:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:52,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:52,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174272192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:52,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174272219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:52,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174272219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:52,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174272219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:52,226 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:52,226 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,227 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:52,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174272299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:52,459 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/dbbf5ef162aa4e0b8a766969c22f52fd as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dbbf5ef162aa4e0b8a766969c22f52fd 2024-12-02T21:16:52,463 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/A of a3e315252bf6ea46b1e790171c7a96a1 into dbbf5ef162aa4e0b8a766969c22f52fd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:52,463 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:52,463 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/A, priority=13, startTime=1733174211610; duration=0sec 2024-12-02T21:16:52,463 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:52,463 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:A 2024-12-02T21:16:52,489 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/20f52c84ddb6440e9216d64eb866aef9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/20f52c84ddb6440e9216d64eb866aef9 2024-12-02T21:16:52,489 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/4fcda08de5cb44c68c015369fcde2a7a 2024-12-02T21:16:52,493 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/C of a3e315252bf6ea46b1e790171c7a96a1 into 20f52c84ddb6440e9216d64eb866aef9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:52,493 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:52,493 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/C, priority=13, startTime=1733174211610; duration=0sec 2024-12-02T21:16:52,494 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:52,494 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:C 2024-12-02T21:16:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/02b68b26985b4dbeb73075e745e2527e is 50, key is test_row_0/C:col10/1733174210038/Put/seqid=0 2024-12-02T21:16:52,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742160_1336 (size=12001) 2024-12-02T21:16:52,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174272509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174272829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T21:16:52,904 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/02b68b26985b4dbeb73075e745e2527e 2024-12-02T21:16:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/9ecb234774e94c04b6e5250f1e268cc4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9ecb234774e94c04b6e5250f1e268cc4 2024-12-02T21:16:52,913 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9ecb234774e94c04b6e5250f1e268cc4, entries=150, sequenceid=96, filesize=30.2 K 2024-12-02T21:16:52,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/4fcda08de5cb44c68c015369fcde2a7a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4fcda08de5cb44c68c015369fcde2a7a 2024-12-02T21:16:52,920 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4fcda08de5cb44c68c015369fcde2a7a, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T21:16:52,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/02b68b26985b4dbeb73075e745e2527e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/02b68b26985b4dbeb73075e745e2527e 2024-12-02T21:16:52,926 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/02b68b26985b4dbeb73075e745e2527e, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T21:16:52,928 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a3e315252bf6ea46b1e790171c7a96a1 in 1312ms, sequenceid=96, compaction requested=false 2024-12-02T21:16:52,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:52,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:52,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-02T21:16:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-02T21:16:52,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-02T21:16:52,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2280 sec 2024-12-02T21:16:52,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.2360 sec 2024-12-02T21:16:53,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:16:53,356 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T21:16:53,356 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T21:16:53,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:53,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b8adb12502604d4dbc981f83889ba4aa_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174213355/Put/seqid=0 2024-12-02T21:16:53,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742161_1337 (size=14594) 2024-12-02T21:16:53,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174213406, max timestamp=1733174213516Num uniques 859; 2024-12-02T21:16:53,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:53,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174213406, max timestamp=1733174213516Num uniques 859; 2024-12-02T21:16:53,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:53,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174213406, max timestamp=1733174213516Num uniques 859; 2024-12-02T21:16:53,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:53,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174273690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:53,781 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:53,786 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b8adb12502604d4dbc981f83889ba4aa_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b8adb12502604d4dbc981f83889ba4aa_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:53,789 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/85bda578c27f476fa0ae81266648ae8e, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:53,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/85bda578c27f476fa0ae81266648ae8e is 175, key is test_row_0/A:col10/1733174213355/Put/seqid=0 2024-12-02T21:16:53,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174273808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:53,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742162_1338 (size=39549) 2024-12-02T21:16:53,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/85bda578c27f476fa0ae81266648ae8e 2024-12-02T21:16:53,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/6f4d3aec08c0416b8a35a8eeafe8fd5f is 50, key is test_row_0/B:col10/1733174213355/Put/seqid=0 2024-12-02T21:16:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742163_1339 (size=12001) 2024-12-02T21:16:53,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/6f4d3aec08c0416b8a35a8eeafe8fd5f 2024-12-02T21:16:53,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/83dc407583544119a6c6b352c05025e6 is 50, key is test_row_0/C:col10/1733174213355/Put/seqid=0 2024-12-02T21:16:53,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742164_1340 (size=12001) 2024-12-02T21:16:54,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174274022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:54,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:54,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/83dc407583544119a6c6b352c05025e6 2024-12-02T21:16:54,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174274240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:54,251 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:54,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/85bda578c27f476fa0ae81266648ae8e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/85bda578c27f476fa0ae81266648ae8e 2024-12-02T21:16:54,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174274249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:54,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174274250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:54,256 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:54,256 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:54,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/85bda578c27f476fa0ae81266648ae8e, entries=200, sequenceid=110, filesize=38.6 K 2024-12-02T21:16:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/6f4d3aec08c0416b8a35a8eeafe8fd5f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/6f4d3aec08c0416b8a35a8eeafe8fd5f 2024-12-02T21:16:54,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/6f4d3aec08c0416b8a35a8eeafe8fd5f, entries=150, sequenceid=110, filesize=11.7 K 2024-12-02T21:16:54,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/83dc407583544119a6c6b352c05025e6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/83dc407583544119a6c6b352c05025e6 2024-12-02T21:16:54,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/83dc407583544119a6c6b352c05025e6, entries=150, sequenceid=110, filesize=11.7 K 2024-12-02T21:16:54,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a3e315252bf6ea46b1e790171c7a96a1 in 923ms, sequenceid=110, compaction requested=true 2024-12-02T21:16:54,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:54,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:54,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:54,279 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:54,280 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:54,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:54,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:54,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:54,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:54,281 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:54,281 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/A is initiating minor compaction (all files) 2024-12-02T21:16:54,281 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/A in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:54,281 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dbbf5ef162aa4e0b8a766969c22f52fd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9ecb234774e94c04b6e5250f1e268cc4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/85bda578c27f476fa0ae81266648ae8e] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=99.2 K 2024-12-02T21:16:54,281 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:54,281 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dbbf5ef162aa4e0b8a766969c22f52fd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9ecb234774e94c04b6e5250f1e268cc4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/85bda578c27f476fa0ae81266648ae8e] 2024-12-02T21:16:54,282 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:54,282 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/B is initiating minor compaction (all files) 2024-12-02T21:16:54,282 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/B in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:54,282 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/a46dd1d942c64afa8642dec4d554fc8b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4fcda08de5cb44c68c015369fcde2a7a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/6f4d3aec08c0416b8a35a8eeafe8fd5f] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=35.3 K 2024-12-02T21:16:54,282 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbbf5ef162aa4e0b8a766969c22f52fd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733174208787 2024-12-02T21:16:54,282 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting a46dd1d942c64afa8642dec4d554fc8b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733174208787 2024-12-02T21:16:54,283 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fcda08de5cb44c68c015369fcde2a7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733174210028 2024-12-02T21:16:54,283 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ecb234774e94c04b6e5250f1e268cc4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733174210028 2024-12-02T21:16:54,283 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f4d3aec08c0416b8a35a8eeafe8fd5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733174212178 2024-12-02T21:16:54,283 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85bda578c27f476fa0ae81266648ae8e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733174212178 2024-12-02T21:16:54,315 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#B#compaction#290 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:54,315 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/cf11be157225497aa2ee871234c0f8ee is 50, key is test_row_0/B:col10/1733174213355/Put/seqid=0 2024-12-02T21:16:54,325 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:54,334 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202e014ed429af84f088d20f20e8f10d95d_a3e315252bf6ea46b1e790171c7a96a1 store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:54,336 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202e014ed429af84f088d20f20e8f10d95d_a3e315252bf6ea46b1e790171c7a96a1, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:54,336 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e014ed429af84f088d20f20e8f10d95d_a3e315252bf6ea46b1e790171c7a96a1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:54,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T21:16:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:54,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742165_1341 (size=12207) 2024-12-02T21:16:54,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742166_1342 (size=4469) 2024-12-02T21:16:54,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028dcc1dbf85414c2b9611ceee19fe62eb_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174213688/Put/seqid=0 2024-12-02T21:16:54,388 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#A#compaction#291 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:54,389 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/e585c3ff6c8741bdba1efd18c2b5060b is 175, key is test_row_0/A:col10/1733174213355/Put/seqid=0 2024-12-02T21:16:54,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742167_1343 (size=31161) 2024-12-02T21:16:54,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742168_1344 (size=14794) 2024-12-02T21:16:54,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:54,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028dcc1dbf85414c2b9611ceee19fe62eb_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028dcc1dbf85414c2b9611ceee19fe62eb_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:54,441 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:54,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa is 175, key is test_row_0/A:col10/1733174213688/Put/seqid=0 2024-12-02T21:16:54,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742169_1345 (size=39749) 2024-12-02T21:16:54,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174274445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:54,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174274568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:54,761 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/cf11be157225497aa2ee871234c0f8ee as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/cf11be157225497aa2ee871234c0f8ee 2024-12-02T21:16:54,766 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/B of a3e315252bf6ea46b1e790171c7a96a1 into cf11be157225497aa2ee871234c0f8ee(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:54,766 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:54,766 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/B, priority=13, startTime=1733174214279; duration=0sec 2024-12-02T21:16:54,766 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:54,766 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:B 2024-12-02T21:16:54,768 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:54,769 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:54,769 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/C is initiating minor compaction (all files) 2024-12-02T21:16:54,769 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/C in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:54,769 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/20f52c84ddb6440e9216d64eb866aef9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/02b68b26985b4dbeb73075e745e2527e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/83dc407583544119a6c6b352c05025e6] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=35.3 K 2024-12-02T21:16:54,770 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f52c84ddb6440e9216d64eb866aef9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733174208787 2024-12-02T21:16:54,771 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b68b26985b4dbeb73075e745e2527e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733174210028 2024-12-02T21:16:54,771 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 83dc407583544119a6c6b352c05025e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733174212178 2024-12-02T21:16:54,778 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:54,778 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/cd681b8255f843809d4680655362281a is 50, key is test_row_0/C:col10/1733174213355/Put/seqid=0 2024-12-02T21:16:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174274789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:54,832 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/e585c3ff6c8741bdba1efd18c2b5060b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/e585c3ff6c8741bdba1efd18c2b5060b 2024-12-02T21:16:54,837 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/A of a3e315252bf6ea46b1e790171c7a96a1 into e585c3ff6c8741bdba1efd18c2b5060b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:54,837 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:54,838 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/A, priority=13, startTime=1733174214279; duration=0sec 2024-12-02T21:16:54,838 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:54,838 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:A 2024-12-02T21:16:54,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742170_1346 (size=12207) 2024-12-02T21:16:54,850 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa 2024-12-02T21:16:54,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/5ee13ecf12a04b9d87dd959505fe9a7b is 50, key is test_row_0/B:col10/1733174213688/Put/seqid=0 2024-12-02T21:16:54,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60834 deadline: 1733174274858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:54,865 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T21:16:54,869 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:54,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-02T21:16:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T21:16:54,871 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:54,871 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:54,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:54,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742171_1347 (size=12151) 2024-12-02T21:16:54,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/5ee13ecf12a04b9d87dd959505fe9a7b 2024-12-02T21:16:54,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/351cf40c88be482792fa8614a65225b9 is 50, key is test_row_0/C:col10/1733174213688/Put/seqid=0 2024-12-02T21:16:54,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742172_1348 (size=12151) 2024-12-02T21:16:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T21:16:55,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T21:16:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:55,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:55,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174275110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:55,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T21:16:55,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:55,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:55,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:55,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:55,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T21:16:55,245 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/cd681b8255f843809d4680655362281a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/cd681b8255f843809d4680655362281a 2024-12-02T21:16:55,252 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/C of a3e315252bf6ea46b1e790171c7a96a1 into cd681b8255f843809d4680655362281a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:55,252 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:55,252 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/C, priority=13, startTime=1733174214280; duration=0sec 2024-12-02T21:16:55,253 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:55,253 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:C 2024-12-02T21:16:55,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/351cf40c88be482792fa8614a65225b9 2024-12-02T21:16:55,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa 2024-12-02T21:16:55,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa, entries=200, sequenceid=147, filesize=38.8 K 2024-12-02T21:16:55,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/5ee13ecf12a04b9d87dd959505fe9a7b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/5ee13ecf12a04b9d87dd959505fe9a7b 2024-12-02T21:16:55,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/5ee13ecf12a04b9d87dd959505fe9a7b, entries=150, sequenceid=147, filesize=11.9 K 2024-12-02T21:16:55,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/351cf40c88be482792fa8614a65225b9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/351cf40c88be482792fa8614a65225b9 2024-12-02T21:16:55,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/351cf40c88be482792fa8614a65225b9, entries=150, sequenceid=147, filesize=11.9 K 2024-12-02T21:16:55,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a3e315252bf6ea46b1e790171c7a96a1 in 973ms, sequenceid=147, compaction requested=false 2024-12-02T21:16:55,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:55,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T21:16:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:55,332 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:16:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:55,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024309f699d79847b09ca38f0f6f8c72d6_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174214439/Put/seqid=0 2024-12-02T21:16:55,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742173_1349 (size=12304) 2024-12-02T21:16:55,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,389 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024309f699d79847b09ca38f0f6f8c72d6_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024309f699d79847b09ca38f0f6f8c72d6_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/1252ea2334c444478ed53e1ceeffe848, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/1252ea2334c444478ed53e1ceeffe848 is 175, key is test_row_0/A:col10/1733174214439/Put/seqid=0 2024-12-02T21:16:55,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742174_1350 (size=31105) 2024-12-02T21:16:55,434 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/1252ea2334c444478ed53e1ceeffe848 2024-12-02T21:16:55,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ab07fcea69854f4a8c53ece92c84aa5d is 50, key is test_row_0/B:col10/1733174214439/Put/seqid=0 2024-12-02T21:16:55,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742175_1351 (size=12151) 2024-12-02T21:16:55,481 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ab07fcea69854f4a8c53ece92c84aa5d 2024-12-02T21:16:55,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/6db97dd1cbdd4928abae95f4c41fce95 is 50, key is test_row_0/C:col10/1733174214439/Put/seqid=0 2024-12-02T21:16:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T21:16:55,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742176_1352 (size=12151) 2024-12-02T21:16:55,531 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/6db97dd1cbdd4928abae95f4c41fce95 2024-12-02T21:16:55,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/1252ea2334c444478ed53e1ceeffe848 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/1252ea2334c444478ed53e1ceeffe848 2024-12-02T21:16:55,544 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/1252ea2334c444478ed53e1ceeffe848, entries=150, sequenceid=160, filesize=30.4 K 2024-12-02T21:16:55,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ab07fcea69854f4a8c53ece92c84aa5d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ab07fcea69854f4a8c53ece92c84aa5d 2024-12-02T21:16:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,548 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ab07fcea69854f4a8c53ece92c84aa5d, entries=150, sequenceid=160, filesize=11.9 K 2024-12-02T21:16:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/6db97dd1cbdd4928abae95f4c41fce95 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6db97dd1cbdd4928abae95f4c41fce95 2024-12-02T21:16:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,570 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6db97dd1cbdd4928abae95f4c41fce95, entries=150, sequenceid=160, filesize=11.9 K 2024-12-02T21:16:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,571 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for a3e315252bf6ea46b1e790171c7a96a1 in 239ms, sequenceid=160, compaction requested=true 2024-12-02T21:16:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-02T21:16:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-02T21:16:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-02T21:16:55,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 702 msec 2024-12-02T21:16:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 707 msec 2024-12-02T21:16:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:55,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:16:55,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:55,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:55,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:55,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:55,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:55,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:55,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202604c920d857c429db4b981454c880661_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174215822/Put/seqid=0 2024-12-02T21:16:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742177_1353 (size=14794) 2024-12-02T21:16:55,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:55,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202604c920d857c429db4b981454c880661_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202604c920d857c429db4b981454c880661_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:55,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/f65440cd32c245d58dc0d4978bb1f689, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:55,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/f65440cd32c245d58dc0d4978bb1f689 is 175, key is test_row_0/A:col10/1733174215822/Put/seqid=0 2024-12-02T21:16:55,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742178_1354 (size=39749) 2024-12-02T21:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T21:16:56,009 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:56,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-02T21:16:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T21:16:56,011 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:56,011 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:56,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:56,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174215863, max timestamp=1733174216007Num uniques 859; 2024-12-02T21:16:56,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174215863, max timestamp=1733174216007Num uniques 859; 2024-12-02T21:16:56,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174215863, max timestamp=1733174216007Num uniques 859; 2024-12-02T21:16:56,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:56,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:56,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:16:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T21:16:56,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T21:16:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:56,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:56,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174276198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:56,298 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/f65440cd32c245d58dc0d4978bb1f689 2024-12-02T21:16:56,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/27e1d389393d4a73b9a257d946d660aa is 50, key is test_row_0/B:col10/1733174215822/Put/seqid=0 2024-12-02T21:16:56,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T21:16:56,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:56,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:56,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:56,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:56,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:56,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174276319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T21:16:56,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742179_1355 (size=12151) 2024-12-02T21:16:56,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/27e1d389393d4a73b9a257d946d660aa 2024-12-02T21:16:56,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/56135e8c8b344abcab79c6014ebf2fd6 is 50, key is test_row_0/C:col10/1733174215822/Put/seqid=0 2024-12-02T21:16:56,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742180_1356 (size=12151) 2024-12-02T21:16:56,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/56135e8c8b344abcab79c6014ebf2fd6 2024-12-02T21:16:56,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/f65440cd32c245d58dc0d4978bb1f689 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/f65440cd32c245d58dc0d4978bb1f689 2024-12-02T21:16:56,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/f65440cd32c245d58dc0d4978bb1f689, entries=200, sequenceid=170, filesize=38.8 K 2024-12-02T21:16:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/27e1d389393d4a73b9a257d946d660aa as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/27e1d389393d4a73b9a257d946d660aa 2024-12-02T21:16:56,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/27e1d389393d4a73b9a257d946d660aa, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T21:16:56,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/56135e8c8b344abcab79c6014ebf2fd6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/56135e8c8b344abcab79c6014ebf2fd6 2024-12-02T21:16:56,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/56135e8c8b344abcab79c6014ebf2fd6, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T21:16:56,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a3e315252bf6ea46b1e790171c7a96a1 in 562ms, sequenceid=170, compaction requested=true 2024-12-02T21:16:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:56,394 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:16:56,394 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:16:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:56,396 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:16:56,396 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141764 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:16:56,396 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/B is initiating minor compaction (all files) 2024-12-02T21:16:56,396 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/A is initiating minor compaction (all files) 2024-12-02T21:16:56,396 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/A in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:56,396 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/B in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:56,396 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/cf11be157225497aa2ee871234c0f8ee, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/5ee13ecf12a04b9d87dd959505fe9a7b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ab07fcea69854f4a8c53ece92c84aa5d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/27e1d389393d4a73b9a257d946d660aa] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=47.5 K 2024-12-02T21:16:56,396 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/e585c3ff6c8741bdba1efd18c2b5060b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/1252ea2334c444478ed53e1ceeffe848, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/f65440cd32c245d58dc0d4978bb1f689] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=138.4 K 2024-12-02T21:16:56,396 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:56,396 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/e585c3ff6c8741bdba1efd18c2b5060b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/1252ea2334c444478ed53e1ceeffe848, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/f65440cd32c245d58dc0d4978bb1f689] 2024-12-02T21:16:56,396 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting cf11be157225497aa2ee871234c0f8ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733174212178 2024-12-02T21:16:56,397 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e585c3ff6c8741bdba1efd18c2b5060b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733174212178 2024-12-02T21:16:56,397 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee13ecf12a04b9d87dd959505fe9a7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733174213673 2024-12-02T21:16:56,397 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ab07fcea69854f4a8c53ece92c84aa5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733174214372 2024-12-02T21:16:56,398 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 27e1d389393d4a73b9a257d946d660aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733174215750 2024-12-02T21:16:56,398 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733174213659 2024-12-02T21:16:56,407 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1252ea2334c444478ed53e1ceeffe848, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733174214372 2024-12-02T21:16:56,407 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65440cd32c245d58dc0d4978bb1f689, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733174215750 2024-12-02T21:16:56,438 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#B#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:56,439 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/9faec3c572cc49cd8b7ce8f3c76cf147 is 50, key is test_row_0/B:col10/1733174215822/Put/seqid=0 2024-12-02T21:16:56,444 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:56,457 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202237e232b97cd4bae9e8adb0b40b7d81b_a3e315252bf6ea46b1e790171c7a96a1 store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:56,459 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202237e232b97cd4bae9e8adb0b40b7d81b_a3e315252bf6ea46b1e790171c7a96a1, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:56,459 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202237e232b97cd4bae9e8adb0b40b7d81b_a3e315252bf6ea46b1e790171c7a96a1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:56,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742181_1357 (size=12493) 2024-12-02T21:16:56,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T21:16:56,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:56,470 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T21:16:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:56,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020a0ba7cc278840878071fd20df980410_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174216076/Put/seqid=0 2024-12-02T21:16:56,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742182_1358 (size=4469) 2024-12-02T21:16:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742183_1359 (size=12304) 2024-12-02T21:16:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:56,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:56,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174276621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T21:16:56,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174276749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:56,872 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/9faec3c572cc49cd8b7ce8f3c76cf147 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9faec3c572cc49cd8b7ce8f3c76cf147 2024-12-02T21:16:56,878 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/B of a3e315252bf6ea46b1e790171c7a96a1 into 9faec3c572cc49cd8b7ce8f3c76cf147(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:56,878 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:56,879 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/B, priority=12, startTime=1733174216394; duration=0sec 2024-12-02T21:16:56,879 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:56,879 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:B 2024-12-02T21:16:56,879 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:16:56,880 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:16:56,880 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/C is initiating minor compaction (all files) 2024-12-02T21:16:56,880 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/C in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:56,880 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/cd681b8255f843809d4680655362281a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/351cf40c88be482792fa8614a65225b9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6db97dd1cbdd4928abae95f4c41fce95, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/56135e8c8b344abcab79c6014ebf2fd6] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=47.5 K 2024-12-02T21:16:56,880 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting cd681b8255f843809d4680655362281a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733174212178 2024-12-02T21:16:56,880 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 351cf40c88be482792fa8614a65225b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733174213673 2024-12-02T21:16:56,881 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6db97dd1cbdd4928abae95f4c41fce95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733174214372 2024-12-02T21:16:56,881 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 56135e8c8b344abcab79c6014ebf2fd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733174215750 2024-12-02T21:16:56,891 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:56,892 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/7ca03bbab9054c2298d0a7a9eb6442d9 is 50, key is test_row_0/C:col10/1733174215822/Put/seqid=0 2024-12-02T21:16:56,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742184_1360 (size=12493) 2024-12-02T21:16:56,900 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#A#compaction#303 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:56,900 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/a09af92fe4394a19a980e1ab45b1d445 is 175, key is test_row_0/A:col10/1733174215822/Put/seqid=0 2024-12-02T21:16:56,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742185_1361 (size=31447) 2024-12-02T21:16:56,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:56,914 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020a0ba7cc278840878071fd20df980410_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020a0ba7cc278840878071fd20df980410_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:56,914 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/a09af92fe4394a19a980e1ab45b1d445 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a09af92fe4394a19a980e1ab45b1d445 2024-12-02T21:16:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/fe1404c0abdc45f3838a0ea0e70eddd3, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/fe1404c0abdc45f3838a0ea0e70eddd3 is 175, key is test_row_0/A:col10/1733174216076/Put/seqid=0 2024-12-02T21:16:56,925 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/A of a3e315252bf6ea46b1e790171c7a96a1 into a09af92fe4394a19a980e1ab45b1d445(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:56,925 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:56,925 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/A, priority=12, startTime=1733174216394; duration=0sec 2024-12-02T21:16:56,925 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:56,925 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:A 2024-12-02T21:16:56,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742186_1362 (size=31105) 2024-12-02T21:16:56,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174276959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T21:16:57,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174277268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:57,304 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/7ca03bbab9054c2298d0a7a9eb6442d9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/7ca03bbab9054c2298d0a7a9eb6442d9 2024-12-02T21:16:57,308 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/C of a3e315252bf6ea46b1e790171c7a96a1 into 7ca03bbab9054c2298d0a7a9eb6442d9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:57,308 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:57,308 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/C, priority=12, startTime=1733174216394; duration=0sec 2024-12-02T21:16:57,308 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:57,308 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:C 2024-12-02T21:16:57,329 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/fe1404c0abdc45f3838a0ea0e70eddd3 2024-12-02T21:16:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/e9fd76f360474fe9a9091264807c81bf is 50, key is test_row_0/B:col10/1733174216076/Put/seqid=0 2024-12-02T21:16:57,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742187_1363 (size=12151) 2024-12-02T21:16:57,350 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/e9fd76f360474fe9a9091264807c81bf 2024-12-02T21:16:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/db3c4390064e4bd38837bfcca1b34999 is 50, key is test_row_0/C:col10/1733174216076/Put/seqid=0 2024-12-02T21:16:57,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742188_1364 (size=12151) 2024-12-02T21:16:57,768 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/db3c4390064e4bd38837bfcca1b34999 2024-12-02T21:16:57,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/fe1404c0abdc45f3838a0ea0e70eddd3 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/fe1404c0abdc45f3838a0ea0e70eddd3 2024-12-02T21:16:57,780 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/fe1404c0abdc45f3838a0ea0e70eddd3, entries=150, sequenceid=207, filesize=30.4 K 2024-12-02T21:16:57,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/e9fd76f360474fe9a9091264807c81bf as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e9fd76f360474fe9a9091264807c81bf 2024-12-02T21:16:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174277783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:57,792 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e9fd76f360474fe9a9091264807c81bf, entries=150, sequenceid=207, filesize=11.9 K 2024-12-02T21:16:57,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-02T21:16:57,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/db3c4390064e4bd38837bfcca1b34999 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/db3c4390064e4bd38837bfcca1b34999 2024-12-02T21:16:57,802 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/db3c4390064e4bd38837bfcca1b34999, entries=150, sequenceid=207, filesize=11.9 K 2024-12-02T21:16:57,803 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a3e315252bf6ea46b1e790171c7a96a1 in 1334ms, sequenceid=207, compaction requested=false 2024-12-02T21:16:57,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-02T21:16:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-02T21:16:57,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-02T21:16:57,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7940 sec 2024-12-02T21:16:57,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.7970 sec 2024-12-02T21:16:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T21:16:58,159 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:16:58,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:16:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-02T21:16:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T21:16:58,161 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:16:58,161 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:16:58,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:16:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T21:16:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:58,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:16:58,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:58,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:58,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:58,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:58,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:58,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:16:58,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022497c81eeadc47fbb70f2bb2e7e24cb2_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174216618/Put/seqid=0 2024-12-02T21:16:58,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742189_1365 (size=14794) 2024-12-02T21:16:58,282 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:58,285 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022497c81eeadc47fbb70f2bb2e7e24cb2_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022497c81eeadc47fbb70f2bb2e7e24cb2_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:58,286 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/80df58623dc34049ae051dec0cf333fe, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:58,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/80df58623dc34049ae051dec0cf333fe is 175, key is test_row_0/A:col10/1733174216618/Put/seqid=0 2024-12-02T21:16:58,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742190_1366 (size=39749) 2024-12-02T21:16:58,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T21:16:58,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733174218304, max timestamp=1733174218326Num uniques 859; 2024-12-02T21:16:58,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T21:16:58,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174218304, max timestamp=1733174218326Num uniques 859; 2024-12-02T21:16:58,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:16:58,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=223, cellSize=10208, totalHeapSize=23432, min timestamp=1733174218304, max timestamp=1733174218326Num uniques 859; 2024-12-02T21:16:58,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=10208, suffix heap size=117064,new segment heap size=23432 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=223 2024-12-02T21:16:58,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174278383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174278394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174278395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T21:16:58,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:58,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T21:16:58,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174278498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174278509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174278519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T21:16:58,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:58,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,694 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/80df58623dc34049ae051dec0cf333fe 2024-12-02T21:16:58,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ffdfc7b5a20d4e72a302d254eab208a4 is 50, key is test_row_0/B:col10/1733174216618/Put/seqid=0 2024-12-02T21:16:58,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742191_1367 (size=12151) 2024-12-02T21:16:58,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174278708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174278729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174278729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T21:16:58,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:58,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T21:16:58,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174278798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:58,804 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:16:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:58,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T21:16:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:58,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:58,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:59,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174279018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174279039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174279039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T21:16:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:59,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:16:59,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ffdfc7b5a20d4e72a302d254eab208a4 2024-12-02T21:16:59,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/778fa0ed240e458c9e300a51d38bff9d is 50, key is test_row_0/C:col10/1733174216618/Put/seqid=0 2024-12-02T21:16:59,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742192_1368 (size=12151) 2024-12-02T21:16:59,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/778fa0ed240e458c9e300a51d38bff9d 2024-12-02T21:16:59,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/80df58623dc34049ae051dec0cf333fe as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/80df58623dc34049ae051dec0cf333fe 2024-12-02T21:16:59,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/80df58623dc34049ae051dec0cf333fe, entries=200, sequenceid=221, filesize=38.8 K 2024-12-02T21:16:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ffdfc7b5a20d4e72a302d254eab208a4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ffdfc7b5a20d4e72a302d254eab208a4 2024-12-02T21:16:59,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ffdfc7b5a20d4e72a302d254eab208a4, entries=150, sequenceid=221, filesize=11.9 K 2024-12-02T21:16:59,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/778fa0ed240e458c9e300a51d38bff9d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/778fa0ed240e458c9e300a51d38bff9d 2024-12-02T21:16:59,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/778fa0ed240e458c9e300a51d38bff9d, entries=150, sequenceid=221, filesize=11.9 K 2024-12-02T21:16:59,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=137.04 KB/140334 for a3e315252bf6ea46b1e790171c7a96a1 in 907ms, sequenceid=221, compaction requested=true 2024-12-02T21:16:59,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:59,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:16:59,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:59,180 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:59,180 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:59,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:16:59,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:59,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:16:59,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:59,182 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:59,182 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/B is initiating minor compaction (all files) 2024-12-02T21:16:59,182 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/B in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:59,182 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9faec3c572cc49cd8b7ce8f3c76cf147, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e9fd76f360474fe9a9091264807c81bf, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ffdfc7b5a20d4e72a302d254eab208a4] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=35.9 K 2024-12-02T21:16:59,182 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:59,182 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/A is initiating minor compaction (all files) 2024-12-02T21:16:59,182 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/A in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:59,182 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a09af92fe4394a19a980e1ab45b1d445, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/fe1404c0abdc45f3838a0ea0e70eddd3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/80df58623dc34049ae051dec0cf333fe] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=99.9 K 2024-12-02T21:16:59,182 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:59,182 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a09af92fe4394a19a980e1ab45b1d445, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/fe1404c0abdc45f3838a0ea0e70eddd3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/80df58623dc34049ae051dec0cf333fe] 2024-12-02T21:16:59,183 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 9faec3c572cc49cd8b7ce8f3c76cf147, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733174215750 2024-12-02T21:16:59,183 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting a09af92fe4394a19a980e1ab45b1d445, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733174215750 2024-12-02T21:16:59,183 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting e9fd76f360474fe9a9091264807c81bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733174216076 2024-12-02T21:16:59,183 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1404c0abdc45f3838a0ea0e70eddd3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733174216076 2024-12-02T21:16:59,183 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdfc7b5a20d4e72a302d254eab208a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733174216570 2024-12-02T21:16:59,183 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80df58623dc34049ae051dec0cf333fe, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733174216570 2024-12-02T21:16:59,193 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:59,205 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:59,206 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ea440679696942c98bdb56a30abf124f is 50, key is test_row_0/B:col10/1733174216618/Put/seqid=0 2024-12-02T21:16:59,210 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202f4c5c775c06a41659a40f3b9a2195c3d_a3e315252bf6ea46b1e790171c7a96a1 store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:59,212 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202f4c5c775c06a41659a40f3b9a2195c3d_a3e315252bf6ea46b1e790171c7a96a1, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:59,212 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f4c5c775c06a41659a40f3b9a2195c3d_a3e315252bf6ea46b1e790171c7a96a1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:59,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T21:16:59,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:59,232 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=137.04 KB heapSize=386.80 KB 2024-12-02T21:16:59,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:16:59,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:59,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:16:59,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:59,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:16:59,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:16:59,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742193_1369 (size=12595) 2024-12-02T21:16:59,265 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ea440679696942c98bdb56a30abf124f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ea440679696942c98bdb56a30abf124f 2024-12-02T21:16:59,272 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/B of a3e315252bf6ea46b1e790171c7a96a1 into ea440679696942c98bdb56a30abf124f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:59,272 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:59,272 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/B, priority=13, startTime=1733174219180; duration=0sec 2024-12-02T21:16:59,272 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:16:59,272 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:B 2024-12-02T21:16:59,272 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:16:59,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742194_1370 (size=4469) 2024-12-02T21:16:59,274 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:16:59,274 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/C is initiating minor compaction (all files) 2024-12-02T21:16:59,274 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#A#compaction#311 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:59,274 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/C in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:16:59,274 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/7ca03bbab9054c2298d0a7a9eb6442d9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/db3c4390064e4bd38837bfcca1b34999, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/778fa0ed240e458c9e300a51d38bff9d] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=35.9 K 2024-12-02T21:16:59,274 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/5b55062e1ed84fd19ab2b55afbee59c4 is 175, key is test_row_0/A:col10/1733174216618/Put/seqid=0 2024-12-02T21:16:59,275 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca03bbab9054c2298d0a7a9eb6442d9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733174215750 2024-12-02T21:16:59,276 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting db3c4390064e4bd38837bfcca1b34999, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733174216076 2024-12-02T21:16:59,276 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 778fa0ed240e458c9e300a51d38bff9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733174216570 2024-12-02T21:16:59,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026301e21cf10948758b64517e41d17afc_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174218392/Put/seqid=0 2024-12-02T21:16:59,291 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#C#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:16:59,293 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/9190cf766bbb4f938382878bcac5f57d is 50, key is test_row_0/C:col10/1733174216618/Put/seqid=0 2024-12-02T21:16:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T21:16:59,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742195_1371 (size=31549) 2024-12-02T21:16:59,308 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/5b55062e1ed84fd19ab2b55afbee59c4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/5b55062e1ed84fd19ab2b55afbee59c4 2024-12-02T21:16:59,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742196_1372 (size=12304) 2024-12-02T21:16:59,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:16:59,317 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/A of a3e315252bf6ea46b1e790171c7a96a1 into 5b55062e1ed84fd19ab2b55afbee59c4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:59,317 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:59,317 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/A, priority=13, startTime=1733174219180; duration=0sec 2024-12-02T21:16:59,317 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:59,317 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:A 2024-12-02T21:16:59,319 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026301e21cf10948758b64517e41d17afc_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026301e21cf10948758b64517e41d17afc_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/8907fadc973a4c9f9f925cb50346e7b5, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:16:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/8907fadc973a4c9f9f925cb50346e7b5 is 175, key is test_row_0/A:col10/1733174218392/Put/seqid=0 2024-12-02T21:16:59,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742197_1373 (size=12595) 2024-12-02T21:16:59,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742198_1374 (size=31105) 2024-12-02T21:16:59,339 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/9190cf766bbb4f938382878bcac5f57d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/9190cf766bbb4f938382878bcac5f57d 2024-12-02T21:16:59,345 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/C of a3e315252bf6ea46b1e790171c7a96a1 into 9190cf766bbb4f938382878bcac5f57d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:16:59,345 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:16:59,345 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/C, priority=13, startTime=1733174219181; duration=0sec 2024-12-02T21:16:59,345 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:16:59,345 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:C 2024-12-02T21:16:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:16:59,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:16:59,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174279567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174279569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174279569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174279679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174279679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174279679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,735 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/8907fadc973a4c9f9f925cb50346e7b5 2024-12-02T21:16:59,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ed965f20d4bf434e8c1d9dfe0cf1741d is 50, key is test_row_0/B:col10/1733174218392/Put/seqid=0 2024-12-02T21:16:59,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742199_1375 (size=12151) 2024-12-02T21:16:59,754 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ed965f20d4bf434e8c1d9dfe0cf1741d 2024-12-02T21:16:59,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/223c367a63334a4886a093c70bffc91a is 50, key is test_row_0/C:col10/1733174218392/Put/seqid=0 2024-12-02T21:16:59,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742200_1376 (size=12151) 2024-12-02T21:16:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174279888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:16:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174279889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174279888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:16:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:16:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:16:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,182 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/223c367a63334a4886a093c70bffc91a 2024-12-02T21:17:00,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/8907fadc973a4c9f9f925cb50346e7b5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/8907fadc973a4c9f9f925cb50346e7b5 2024-12-02T21:17:00,201 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/8907fadc973a4c9f9f925cb50346e7b5, entries=150, sequenceid=257, filesize=30.4 K 2024-12-02T21:17:00,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ed965f20d4bf434e8c1d9dfe0cf1741d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed965f20d4bf434e8c1d9dfe0cf1741d 2024-12-02T21:17:00,207 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed965f20d4bf434e8c1d9dfe0cf1741d, entries=150, sequenceid=257, filesize=11.9 K 2024-12-02T21:17:00,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/223c367a63334a4886a093c70bffc91a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/223c367a63334a4886a093c70bffc91a 2024-12-02T21:17:00,215 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/223c367a63334a4886a093c70bffc91a, entries=150, sequenceid=257, filesize=11.9 K 2024-12-02T21:17:00,216 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~137.04 KB/140334, heapSize ~386.75 KB/396032, currentSize=46.96 KB/48090 for a3e315252bf6ea46b1e790171c7a96a1 in 984ms, sequenceid=257, compaction requested=false 2024-12-02T21:17:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-02T21:17:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-02T21:17:00,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-02T21:17:00,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-12-02T21:17:00,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0610 sec 2024-12-02T21:17:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:00,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:17:00,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:17:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:17:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:17:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:00,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028bd45894b915416a925758a5d35a95a8_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174220223/Put/seqid=0 2024-12-02T21:17:00,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742201_1377 (size=17534) 2024-12-02T21:17:00,280 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:00,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028bd45894b915416a925758a5d35a95a8_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028bd45894b915416a925758a5d35a95a8_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:00,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/bd51df34db094a7491fe1066b5e03296, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:00,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/bd51df34db094a7491fe1066b5e03296 is 175, key is test_row_0/A:col10/1733174220223/Put/seqid=0 2024-12-02T21:17:00,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174280286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742202_1378 (size=48639) 2024-12-02T21:17:00,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/bd51df34db094a7491fe1066b5e03296 2024-12-02T21:17:00,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174280290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174280291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/4c1625093ca143618d390f35150eb834 is 50, key is test_row_0/B:col10/1733174220223/Put/seqid=0 2024-12-02T21:17:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T21:17:00,309 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:00,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-02T21:17:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T21:17:00,315 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:00,315 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:00,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:00,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742203_1379 (size=12301) 2024-12-02T21:17:00,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174280399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T21:17:00,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174280409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174280409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T21:17:00,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:00,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:00,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:00,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174280619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T21:17:00,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174280629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174280629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T21:17:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:00,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/4c1625093ca143618d390f35150eb834 2024-12-02T21:17:00,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/aa71309ed3ce40debaf1867fd8184eba is 50, key is test_row_0/C:col10/1733174220223/Put/seqid=0 2024-12-02T21:17:00,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742204_1380 (size=12301) 2024-12-02T21:17:00,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T21:17:00,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:00,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:00,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:00,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174280829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,836 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174280929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T21:17:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:00,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T21:17:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:00,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174280939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174280940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:01,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T21:17:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:01,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:01,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/aa71309ed3ce40debaf1867fd8184eba 2024-12-02T21:17:01,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/bd51df34db094a7491fe1066b5e03296 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/bd51df34db094a7491fe1066b5e03296 2024-12-02T21:17:01,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/bd51df34db094a7491fe1066b5e03296, entries=250, sequenceid=273, filesize=47.5 K 2024-12-02T21:17:01,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/4c1625093ca143618d390f35150eb834 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4c1625093ca143618d390f35150eb834 2024-12-02T21:17:01,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4c1625093ca143618d390f35150eb834, entries=150, sequenceid=273, filesize=12.0 K 2024-12-02T21:17:01,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/aa71309ed3ce40debaf1867fd8184eba as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/aa71309ed3ce40debaf1867fd8184eba 2024-12-02T21:17:01,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/aa71309ed3ce40debaf1867fd8184eba, entries=150, sequenceid=273, filesize=12.0 K 2024-12-02T21:17:01,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a3e315252bf6ea46b1e790171c7a96a1 in 931ms, sequenceid=273, compaction requested=true 2024-12-02T21:17:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:01,156 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:01,156 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:01,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:01,159 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111293 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:01,159 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:01,159 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/B is initiating minor compaction (all files) 2024-12-02T21:17:01,159 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/A is initiating minor compaction (all files) 2024-12-02T21:17:01,159 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/B in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:01,159 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/A in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:01,159 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ea440679696942c98bdb56a30abf124f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed965f20d4bf434e8c1d9dfe0cf1741d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4c1625093ca143618d390f35150eb834] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=36.2 K 2024-12-02T21:17:01,159 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/5b55062e1ed84fd19ab2b55afbee59c4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/8907fadc973a4c9f9f925cb50346e7b5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/bd51df34db094a7491fe1066b5e03296] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=108.7 K 2024-12-02T21:17:01,160 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:01,160 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/5b55062e1ed84fd19ab2b55afbee59c4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/8907fadc973a4c9f9f925cb50346e7b5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/bd51df34db094a7491fe1066b5e03296] 2024-12-02T21:17:01,160 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ea440679696942c98bdb56a30abf124f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733174216570 2024-12-02T21:17:01,160 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b55062e1ed84fd19ab2b55afbee59c4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733174216570 2024-12-02T21:17:01,160 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ed965f20d4bf434e8c1d9dfe0cf1741d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733174218374 2024-12-02T21:17:01,160 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8907fadc973a4c9f9f925cb50346e7b5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733174218374 2024-12-02T21:17:01,160 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c1625093ca143618d390f35150eb834, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174219565 2024-12-02T21:17:01,160 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd51df34db094a7491fe1066b5e03296, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174219565 2024-12-02T21:17:01,173 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:01,174 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:01,175 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ed0ae257884c49d9909bd9d934c46cf4 is 50, key is test_row_0/B:col10/1733174220223/Put/seqid=0 2024-12-02T21:17:01,179 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202a30f612555bf44f79fd1cab0d205f4b4_a3e315252bf6ea46b1e790171c7a96a1 store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:01,180 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202a30f612555bf44f79fd1cab0d205f4b4_a3e315252bf6ea46b1e790171c7a96a1, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:01,181 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a30f612555bf44f79fd1cab0d205f4b4_a3e315252bf6ea46b1e790171c7a96a1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:01,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742205_1381 (size=12847) 2024-12-02T21:17:01,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742206_1382 (size=4469) 2024-12-02T21:17:01,190 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#A#compaction#320 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:01,191 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/29ad91e212224c0488208be35f4c5023 is 175, key is test_row_0/A:col10/1733174220223/Put/seqid=0 2024-12-02T21:17:01,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742207_1383 (size=31801) 2024-12-02T21:17:01,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T21:17:01,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:01,245 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:01,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029a5eb261be8d414c87abae568e9aa86e_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174220284/Put/seqid=0 2024-12-02T21:17:01,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742208_1384 (size=12454) 2024-12-02T21:17:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:01,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T21:17:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174281463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:01,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174281464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174281465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:01,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174281568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:01,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174281578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:01,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174281579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:01,585 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/ed0ae257884c49d9909bd9d934c46cf4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed0ae257884c49d9909bd9d934c46cf4 2024-12-02T21:17:01,588 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/B of a3e315252bf6ea46b1e790171c7a96a1 into ed0ae257884c49d9909bd9d934c46cf4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:01,588 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:01,588 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/B, priority=13, startTime=1733174221156; duration=0sec 2024-12-02T21:17:01,588 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:01,588 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:B 2024-12-02T21:17:01,588 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:01,589 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:01,589 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/C is initiating minor compaction (all files) 2024-12-02T21:17:01,589 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/C in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:01,589 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/9190cf766bbb4f938382878bcac5f57d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/223c367a63334a4886a093c70bffc91a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/aa71309ed3ce40debaf1867fd8184eba] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=36.2 K 2024-12-02T21:17:01,589 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 9190cf766bbb4f938382878bcac5f57d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733174216570 2024-12-02T21:17:01,590 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 223c367a63334a4886a093c70bffc91a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733174218374 2024-12-02T21:17:01,590 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting aa71309ed3ce40debaf1867fd8184eba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174219565 2024-12-02T21:17:01,595 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:01,595 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/1957b55c7cec45b59652d87031ec2f91 is 50, key is test_row_0/C:col10/1733174220223/Put/seqid=0 2024-12-02T21:17:01,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742209_1385 (size=12847) 2024-12-02T21:17:01,604 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/29ad91e212224c0488208be35f4c5023 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/29ad91e212224c0488208be35f4c5023 2024-12-02T21:17:01,608 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/A of a3e315252bf6ea46b1e790171c7a96a1 into 29ad91e212224c0488208be35f4c5023(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:01,608 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:01,608 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/A, priority=13, startTime=1733174221156; duration=0sec 2024-12-02T21:17:01,608 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:01,608 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:A 2024-12-02T21:17:01,608 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/1957b55c7cec45b59652d87031ec2f91 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/1957b55c7cec45b59652d87031ec2f91 2024-12-02T21:17:01,614 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/C of a3e315252bf6ea46b1e790171c7a96a1 into 1957b55c7cec45b59652d87031ec2f91(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:01,614 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:01,614 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/C, priority=13, startTime=1733174221156; duration=0sec 2024-12-02T21:17:01,614 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:01,614 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:C 2024-12-02T21:17:01,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:01,662 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029a5eb261be8d414c87abae568e9aa86e_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029a5eb261be8d414c87abae568e9aa86e_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:01,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/318410b604d64455b2fc39fe87eeac6b, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:01,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/318410b604d64455b2fc39fe87eeac6b is 175, key is test_row_0/A:col10/1733174220284/Put/seqid=0 2024-12-02T21:17:01,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742210_1386 (size=31255) 2024-12-02T21:17:01,669 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/318410b604d64455b2fc39fe87eeac6b 2024-12-02T21:17:01,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/e31fba67c1d2484886b4b932c5e192df is 50, key is test_row_0/B:col10/1733174220284/Put/seqid=0 2024-12-02T21:17:01,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742211_1387 (size=12301) 2024-12-02T21:17:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174281778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174281789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174281789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,079 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/e31fba67c1d2484886b4b932c5e192df 2024-12-02T21:17:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/2854f292164f4b92809392d74c0915b1 is 50, key is test_row_0/C:col10/1733174220284/Put/seqid=0 2024-12-02T21:17:02,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174282088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742212_1388 (size=12301) 2024-12-02T21:17:02,093 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/2854f292164f4b92809392d74c0915b1 2024-12-02T21:17:02,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/318410b604d64455b2fc39fe87eeac6b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/318410b604d64455b2fc39fe87eeac6b 2024-12-02T21:17:02,099 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/318410b604d64455b2fc39fe87eeac6b, entries=150, sequenceid=293, filesize=30.5 K 2024-12-02T21:17:02,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/e31fba67c1d2484886b4b932c5e192df as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e31fba67c1d2484886b4b932c5e192df 2024-12-02T21:17:02,103 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e31fba67c1d2484886b4b932c5e192df, entries=150, sequenceid=293, filesize=12.0 K 2024-12-02T21:17:02,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/2854f292164f4b92809392d74c0915b1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/2854f292164f4b92809392d74c0915b1 2024-12-02T21:17:02,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174282099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174282099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,110 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/2854f292164f4b92809392d74c0915b1, entries=150, sequenceid=293, filesize=12.0 K 2024-12-02T21:17:02,111 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a3e315252bf6ea46b1e790171c7a96a1 in 867ms, sequenceid=293, compaction requested=false 2024-12-02T21:17:02,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:02,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:02,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-02T21:17:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-02T21:17:02,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-02T21:17:02,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8010 sec 2024-12-02T21:17:02,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.8110 sec 2024-12-02T21:17:02,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T21:17:02,459 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:02,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-02T21:17:02,460 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T21:17:02,461 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:02,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T21:17:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:02,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T21:17:02,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:17:02,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:02,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:17:02,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:02,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:17:02,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:02,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120244d197e042344f5a9d981e14fae28cca_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174221464/Put/seqid=0 2024-12-02T21:17:02,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:02,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:02,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:02,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742213_1389 (size=14994) 2024-12-02T21:17:02,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174282648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174282655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174282656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:02,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:02,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174282769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T21:17:02,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174282779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174282779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:02,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:02,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:02,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174282989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:03,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174282999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:03,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174282999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:03,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:03,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120244d197e042344f5a9d981e14fae28cca_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120244d197e042344f5a9d981e14fae28cca_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:03,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/59182562e9104a5789d0c9c8f5e05d7d, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:03,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/59182562e9104a5789d0c9c8f5e05d7d is 175, key is test_row_0/A:col10/1733174221464/Put/seqid=0 2024-12-02T21:17:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742214_1390 (size=39949) 2024-12-02T21:17:03,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T21:17:03,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:03,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:03,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174283298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174283308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174283309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:03,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,441 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/59182562e9104a5789d0c9c8f5e05d7d 2024-12-02T21:17:03,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/0643fe652be54823a33e552d655e5d3d is 50, key is test_row_0/B:col10/1733174221464/Put/seqid=0 2024-12-02T21:17:03,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742215_1391 (size=12301) 2024-12-02T21:17:03,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:03,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:03,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T21:17:03,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:03,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:03,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174283808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:03,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174283829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174283829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:03,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/0643fe652be54823a33e552d655e5d3d 2024-12-02T21:17:03,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/189f39fcf2614db39e3c9d9a193f5195 is 50, key is test_row_0/C:col10/1733174221464/Put/seqid=0 2024-12-02T21:17:03,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742216_1392 (size=12301) 2024-12-02T21:17:03,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:03,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:04,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:04,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:04,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:04,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:04,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:04,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:04,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/189f39fcf2614db39e3c9d9a193f5195 2024-12-02T21:17:04,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/59182562e9104a5789d0c9c8f5e05d7d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/59182562e9104a5789d0c9c8f5e05d7d 2024-12-02T21:17:04,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/59182562e9104a5789d0c9c8f5e05d7d, entries=200, sequenceid=309, filesize=39.0 K 2024-12-02T21:17:04,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/0643fe652be54823a33e552d655e5d3d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/0643fe652be54823a33e552d655e5d3d 2024-12-02T21:17:04,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/0643fe652be54823a33e552d655e5d3d, entries=150, sequenceid=309, filesize=12.0 K 2024-12-02T21:17:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/189f39fcf2614db39e3c9d9a193f5195 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/189f39fcf2614db39e3c9d9a193f5195 2024-12-02T21:17:04,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/189f39fcf2614db39e3c9d9a193f5195, entries=150, sequenceid=309, filesize=12.0 K 2024-12-02T21:17:04,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a3e315252bf6ea46b1e790171c7a96a1 in 1674ms, sequenceid=309, compaction requested=true 2024-12-02T21:17:04,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:04,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:04,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:04,275 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:04,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:04,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:04,275 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:04,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:04,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:04,276 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:04,276 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:04,276 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/A is initiating minor compaction (all files) 2024-12-02T21:17:04,276 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/B is initiating minor compaction (all files) 2024-12-02T21:17:04,276 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/B in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:04,276 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/A in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:04,277 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed0ae257884c49d9909bd9d934c46cf4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e31fba67c1d2484886b4b932c5e192df, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/0643fe652be54823a33e552d655e5d3d] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=36.6 K 2024-12-02T21:17:04,277 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/29ad91e212224c0488208be35f4c5023, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/318410b604d64455b2fc39fe87eeac6b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/59182562e9104a5789d0c9c8f5e05d7d] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=100.6 K 2024-12-02T21:17:04,277 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:04,277 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/29ad91e212224c0488208be35f4c5023, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/318410b604d64455b2fc39fe87eeac6b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/59182562e9104a5789d0c9c8f5e05d7d] 2024-12-02T21:17:04,277 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ad91e212224c0488208be35f4c5023, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174219565 2024-12-02T21:17:04,277 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ed0ae257884c49d9909bd9d934c46cf4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174219565 2024-12-02T21:17:04,277 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting e31fba67c1d2484886b4b932c5e192df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733174220257 2024-12-02T21:17:04,277 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318410b604d64455b2fc39fe87eeac6b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733174220257 2024-12-02T21:17:04,278 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59182562e9104a5789d0c9c8f5e05d7d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733174221458 2024-12-02T21:17:04,278 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 0643fe652be54823a33e552d655e5d3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733174221463 2024-12-02T21:17:04,281 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:04,283 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#B#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:04,283 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202544fe78156dd44d0bb4c0de48a2023f8_a3e315252bf6ea46b1e790171c7a96a1 store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:04,283 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/3da80309054043ab8a8c361e4287ab04 is 50, key is test_row_0/B:col10/1733174221464/Put/seqid=0 2024-12-02T21:17:04,285 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202544fe78156dd44d0bb4c0de48a2023f8_a3e315252bf6ea46b1e790171c7a96a1, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:04,285 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202544fe78156dd44d0bb4c0de48a2023f8_a3e315252bf6ea46b1e790171c7a96a1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:04,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742217_1393 (size=12949) 2024-12-02T21:17:04,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742218_1394 (size=4469) 2024-12-02T21:17:04,298 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#A#compaction#330 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:04,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T21:17:04,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:04,298 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T21:17:04,299 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/7d7d0feeda514815a1e0fd2ba82865ef is 175, key is test_row_0/A:col10/1733174221464/Put/seqid=0 2024-12-02T21:17:04,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:17:04,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:04,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:17:04,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:04,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:17:04,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:04,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742219_1395 (size=31903) 2024-12-02T21:17:04,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120240d188302f20428497477ef417c8695b_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174222654/Put/seqid=0 2024-12-02T21:17:04,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742220_1396 (size=12454) 2024-12-02T21:17:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T21:17:04,690 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/3da80309054043ab8a8c361e4287ab04 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/3da80309054043ab8a8c361e4287ab04 2024-12-02T21:17:04,694 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/B of a3e315252bf6ea46b1e790171c7a96a1 into 3da80309054043ab8a8c361e4287ab04(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:04,694 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:04,694 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/B, priority=13, startTime=1733174224275; duration=0sec 2024-12-02T21:17:04,694 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:04,694 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:B 2024-12-02T21:17:04,694 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:04,695 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:04,695 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/C is initiating minor compaction (all files) 2024-12-02T21:17:04,695 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/C in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:04,695 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/1957b55c7cec45b59652d87031ec2f91, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/2854f292164f4b92809392d74c0915b1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/189f39fcf2614db39e3c9d9a193f5195] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=36.6 K 2024-12-02T21:17:04,695 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 1957b55c7cec45b59652d87031ec2f91, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174219565 2024-12-02T21:17:04,695 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 2854f292164f4b92809392d74c0915b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733174220257 2024-12-02T21:17:04,696 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 189f39fcf2614db39e3c9d9a193f5195, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733174221463 2024-12-02T21:17:04,702 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:04,702 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/33dc8f9b3cae4365952d35e3c00605fa is 50, key is test_row_0/C:col10/1733174221464/Put/seqid=0 2024-12-02T21:17:04,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742221_1397 (size=12949) 2024-12-02T21:17:04,710 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/33dc8f9b3cae4365952d35e3c00605fa as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/33dc8f9b3cae4365952d35e3c00605fa 2024-12-02T21:17:04,710 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/7d7d0feeda514815a1e0fd2ba82865ef as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7d7d0feeda514815a1e0fd2ba82865ef 2024-12-02T21:17:04,715 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/C of a3e315252bf6ea46b1e790171c7a96a1 into 33dc8f9b3cae4365952d35e3c00605fa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:04,715 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:04,715 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/C, priority=13, startTime=1733174224275; duration=0sec 2024-12-02T21:17:04,715 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:04,715 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:C 2024-12-02T21:17:04,716 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/A of a3e315252bf6ea46b1e790171c7a96a1 into 7d7d0feeda514815a1e0fd2ba82865ef(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:04,716 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:04,716 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/A, priority=13, startTime=1733174224275; duration=0sec 2024-12-02T21:17:04,716 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:04,716 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:A 2024-12-02T21:17:04,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:04,720 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120240d188302f20428497477ef417c8695b_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120240d188302f20428497477ef417c8695b_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:04,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/7e4d2606c5164ec3abb90e88e5a23793, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:04,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/7e4d2606c5164ec3abb90e88e5a23793 is 175, key is test_row_0/A:col10/1733174222654/Put/seqid=0 2024-12-02T21:17:04,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742222_1398 (size=31255) 2024-12-02T21:17:04,731 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/7e4d2606c5164ec3abb90e88e5a23793 2024-12-02T21:17:04,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/7b7ea6f25dd14062acc1fa723cb8828d is 50, key is test_row_0/B:col10/1733174222654/Put/seqid=0 2024-12-02T21:17:04,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742223_1399 (size=12301) 2024-12-02T21:17:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:04,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. as already flushing 2024-12-02T21:17:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174284852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174284853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:04,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174284855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174284856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:04,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60834 deadline: 1733174284948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:04,953 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in a3e315252bf6ea46b1e790171c7a96a1 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18342 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:04,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174284969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:04,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:04,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174284969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174284969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:04,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174284969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:05,143 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/7b7ea6f25dd14062acc1fa723cb8828d 2024-12-02T21:17:05,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/3bd0c2c3daf74189a04a7a5ccedb8ae9 is 50, key is test_row_0/C:col10/1733174222654/Put/seqid=0 2024-12-02T21:17:05,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742224_1400 (size=12301) 2024-12-02T21:17:05,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174285179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174285179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:05,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174285179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:05,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174285179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:05,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174285488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:05,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174285488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:05,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174285488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:05,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174285489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:05,558 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/3bd0c2c3daf74189a04a7a5ccedb8ae9 2024-12-02T21:17:05,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/7e4d2606c5164ec3abb90e88e5a23793 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7e4d2606c5164ec3abb90e88e5a23793 2024-12-02T21:17:05,563 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7e4d2606c5164ec3abb90e88e5a23793, entries=150, sequenceid=328, filesize=30.5 K 2024-12-02T21:17:05,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/7b7ea6f25dd14062acc1fa723cb8828d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/7b7ea6f25dd14062acc1fa723cb8828d 2024-12-02T21:17:05,566 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/7b7ea6f25dd14062acc1fa723cb8828d, entries=150, sequenceid=328, filesize=12.0 K 2024-12-02T21:17:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/3bd0c2c3daf74189a04a7a5ccedb8ae9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/3bd0c2c3daf74189a04a7a5ccedb8ae9 2024-12-02T21:17:05,570 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/3bd0c2c3daf74189a04a7a5ccedb8ae9, entries=150, sequenceid=328, filesize=12.0 K 2024-12-02T21:17:05,571 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a3e315252bf6ea46b1e790171c7a96a1 in 1273ms, sequenceid=328, compaction requested=false 2024-12-02T21:17:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-02T21:17:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-02T21:17:05,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-02T21:17:05,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1110 sec 2024-12-02T21:17:05,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.1150 sec 2024-12-02T21:17:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:06,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:17:06,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:17:06,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:06,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:17:06,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:06,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:17:06,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:06,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120223827a5167f04d6ab1759c27b723606a_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174226002/Put/seqid=0 2024-12-02T21:17:06,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742225_1401 (size=12454) 2024-12-02T21:17:06,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174286038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174286040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174286041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174286042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174286159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174286159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174286159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174286159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174286369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174286369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174286369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174286369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,420 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:06,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120223827a5167f04d6ab1759c27b723606a_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120223827a5167f04d6ab1759c27b723606a_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:06,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/eef09f1aa9344fbfb6471d7157cdd6ad, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:06,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/eef09f1aa9344fbfb6471d7157cdd6ad is 175, key is test_row_0/A:col10/1733174226002/Put/seqid=0 2024-12-02T21:17:06,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742226_1402 (size=31255) 2024-12-02T21:17:06,526 INFO [Thread-1470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1470. 2024-12-02T21:17:06,526 DEBUG [Thread-1470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:06,526 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,526 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:06,529 INFO [Thread-1466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1466. 2024-12-02T21:17:06,529 DEBUG [Thread-1466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:06,529 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,529 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:06,532 INFO [Thread-1468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1468. 2024-12-02T21:17:06,532 DEBUG [Thread-1468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:06,532 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,532 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:06,535 INFO [Thread-1462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1462. 2024-12-02T21:17:06,535 DEBUG [Thread-1462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:06,535 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,535 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:06,536 INFO [Thread-1464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1464. 2024-12-02T21:17:06,536 DEBUG [Thread-1464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:06,536 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,536 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:06,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T21:17:06,620 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:06,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174286679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174286679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174286679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174286679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:06,831 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/eef09f1aa9344fbfb6471d7157cdd6ad 2024-12-02T21:17:06,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/e6b32f2b21b14595bea6143ecaebd36a is 50, key is test_row_0/B:col10/1733174226002/Put/seqid=0 2024-12-02T21:17:06,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742227_1403 (size=12301) 2024-12-02T21:17:07,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:07,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:07,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60772 deadline: 1733174287189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60850 deadline: 1733174287189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60792 deadline: 1733174287189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:07,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60802 deadline: 1733174287189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:07,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/e6b32f2b21b14595bea6143ecaebd36a 2024-12-02T21:17:07,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/6c7b766c4d3c4b19aa8f852188b163b4 is 50, key is test_row_0/C:col10/1733174226002/Put/seqid=0 2024-12-02T21:17:07,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742228_1404 (size=12301) 2024-12-02T21:17:07,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/6c7b766c4d3c4b19aa8f852188b163b4 2024-12-02T21:17:07,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/eef09f1aa9344fbfb6471d7157cdd6ad as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/eef09f1aa9344fbfb6471d7157cdd6ad 2024-12-02T21:17:07,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/eef09f1aa9344fbfb6471d7157cdd6ad, entries=150, sequenceid=345, filesize=30.5 K 2024-12-02T21:17:07,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/e6b32f2b21b14595bea6143ecaebd36a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e6b32f2b21b14595bea6143ecaebd36a 2024-12-02T21:17:07,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e6b32f2b21b14595bea6143ecaebd36a, entries=150, sequenceid=345, filesize=12.0 K 2024-12-02T21:17:07,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/6c7b766c4d3c4b19aa8f852188b163b4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6c7b766c4d3c4b19aa8f852188b163b4 2024-12-02T21:17:07,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6c7b766c4d3c4b19aa8f852188b163b4, entries=150, sequenceid=345, filesize=12.0 K 2024-12-02T21:17:07,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a3e315252bf6ea46b1e790171c7a96a1 in 1673ms, sequenceid=345, compaction requested=true 2024-12-02T21:17:07,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:07,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:07,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:07,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:07,676 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:07,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:07,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e315252bf6ea46b1e790171c7a96a1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:07,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:07,676 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:07,677 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:07,677 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:07,677 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/B is initiating minor compaction (all files) 2024-12-02T21:17:07,677 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/A is initiating minor compaction (all files) 2024-12-02T21:17:07,677 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/A in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:07,677 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/B in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:07,678 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7d7d0feeda514815a1e0fd2ba82865ef, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7e4d2606c5164ec3abb90e88e5a23793, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/eef09f1aa9344fbfb6471d7157cdd6ad] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=92.2 K 2024-12-02T21:17:07,678 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/3da80309054043ab8a8c361e4287ab04, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/7b7ea6f25dd14062acc1fa723cb8828d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e6b32f2b21b14595bea6143ecaebd36a] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=36.7 K 2024-12-02T21:17:07,678 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:07,678 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7d7d0feeda514815a1e0fd2ba82865ef, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7e4d2606c5164ec3abb90e88e5a23793, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/eef09f1aa9344fbfb6471d7157cdd6ad] 2024-12-02T21:17:07,678 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da80309054043ab8a8c361e4287ab04, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733174221463 2024-12-02T21:17:07,678 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7d0feeda514815a1e0fd2ba82865ef, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733174221463 2024-12-02T21:17:07,678 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7ea6f25dd14062acc1fa723cb8828d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733174222622 2024-12-02T21:17:07,678 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e4d2606c5164ec3abb90e88e5a23793, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733174222622 2024-12-02T21:17:07,678 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b32f2b21b14595bea6143ecaebd36a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733174224848 2024-12-02T21:17:07,678 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting eef09f1aa9344fbfb6471d7157cdd6ad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733174224848 2024-12-02T21:17:07,684 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#B#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:07,684 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/80feffba9c944b64b4fc531de2b7082c is 50, key is test_row_0/B:col10/1733174226002/Put/seqid=0 2024-12-02T21:17:07,686 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:07,688 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202ff4ab0346e194d3e8bdde6e2761b7868_a3e315252bf6ea46b1e790171c7a96a1 store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:07,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742229_1405 (size=13051) 2024-12-02T21:17:07,695 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/80feffba9c944b64b4fc531de2b7082c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/80feffba9c944b64b4fc531de2b7082c 2024-12-02T21:17:07,700 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202ff4ab0346e194d3e8bdde6e2761b7868_a3e315252bf6ea46b1e790171c7a96a1, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:07,700 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/B of a3e315252bf6ea46b1e790171c7a96a1 into 80feffba9c944b64b4fc531de2b7082c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:07,700 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:07,700 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/B, priority=13, startTime=1733174227676; duration=0sec 2024-12-02T21:17:07,700 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ff4ab0346e194d3e8bdde6e2761b7868_a3e315252bf6ea46b1e790171c7a96a1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:07,700 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:07,700 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:B 2024-12-02T21:17:07,700 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:07,701 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:07,701 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): a3e315252bf6ea46b1e790171c7a96a1/C is initiating minor compaction (all files) 2024-12-02T21:17:07,701 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e315252bf6ea46b1e790171c7a96a1/C in TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:07,701 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/33dc8f9b3cae4365952d35e3c00605fa, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/3bd0c2c3daf74189a04a7a5ccedb8ae9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6c7b766c4d3c4b19aa8f852188b163b4] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp, totalSize=36.7 K 2024-12-02T21:17:07,702 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 33dc8f9b3cae4365952d35e3c00605fa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733174221463 2024-12-02T21:17:07,702 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bd0c2c3daf74189a04a7a5ccedb8ae9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733174222622 2024-12-02T21:17:07,702 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c7b766c4d3c4b19aa8f852188b163b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733174224848 2024-12-02T21:17:07,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742230_1406 (size=4469) 2024-12-02T21:17:07,716 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#C#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:07,716 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/aa10d757349b44f9ba858465afe39e99 is 50, key is test_row_0/C:col10/1733174226002/Put/seqid=0 2024-12-02T21:17:07,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742231_1407 (size=13051) 2024-12-02T21:17:08,116 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e315252bf6ea46b1e790171c7a96a1#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:08,118 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/a6294b1f9b8140a1bdb76563fad7b503 is 175, key is test_row_0/A:col10/1733174226002/Put/seqid=0 2024-12-02T21:17:08,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742232_1408 (size=32005) 2024-12-02T21:17:08,125 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/aa10d757349b44f9ba858465afe39e99 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/aa10d757349b44f9ba858465afe39e99 2024-12-02T21:17:08,131 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/C of a3e315252bf6ea46b1e790171c7a96a1 into aa10d757349b44f9ba858465afe39e99(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:08,131 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:08,131 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/C, priority=13, startTime=1733174227676; duration=0sec 2024-12-02T21:17:08,131 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:08,131 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:C 2024-12-02T21:17:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:08,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T21:17:08,201 INFO [Thread-1451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1451. 2024-12-02T21:17:08,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:17:08,201 DEBUG [Thread-1451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:08,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:17:08,201 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:08,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:17:08,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:17:08,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:17:08,202 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:08,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:17:08,202 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:08,202 INFO [Thread-1455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1455. 2024-12-02T21:17:08,202 DEBUG [Thread-1455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:08,202 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:08,202 INFO [Thread-1453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1453. 2024-12-02T21:17:08,202 DEBUG [Thread-1453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:08,202 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:08,202 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:08,202 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:08,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:08,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:08,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026d1cf2541cbe48709108c49173e5e2dd_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174226004/Put/seqid=0 2024-12-02T21:17:08,210 INFO [Thread-1457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1457. 2024-12-02T21:17:08,210 DEBUG [Thread-1457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:08,210 DEBUG [Thread-1457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:08,210 DEBUG [Thread-1457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:08,210 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:08,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742233_1409 (size=12454) 2024-12-02T21:17:08,531 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/a6294b1f9b8140a1bdb76563fad7b503 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6294b1f9b8140a1bdb76563fad7b503 2024-12-02T21:17:08,538 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e315252bf6ea46b1e790171c7a96a1/A of a3e315252bf6ea46b1e790171c7a96a1 into a6294b1f9b8140a1bdb76563fad7b503(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:08,538 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:08,538 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., storeName=a3e315252bf6ea46b1e790171c7a96a1/A, priority=13, startTime=1733174227676; duration=0sec 2024-12-02T21:17:08,538 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:08,538 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e315252bf6ea46b1e790171c7a96a1:A 2024-12-02T21:17:08,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:08,622 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026d1cf2541cbe48709108c49173e5e2dd_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026d1cf2541cbe48709108c49173e5e2dd_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:08,623 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/dad3c2167a6f485198916a370412a0a8, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:08,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/dad3c2167a6f485198916a370412a0a8 is 175, key is test_row_0/A:col10/1733174226004/Put/seqid=0 2024-12-02T21:17:08,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742234_1410 (size=31255) 2024-12-02T21:17:09,028 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/dad3c2167a6f485198916a370412a0a8 2024-12-02T21:17:09,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/a67de0a3e03f4e49b1c8a2c77b2b5c7c is 50, key is test_row_0/B:col10/1733174226004/Put/seqid=0 2024-12-02T21:17:09,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742235_1411 (size=12301) 2024-12-02T21:17:09,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/a67de0a3e03f4e49b1c8a2c77b2b5c7c 2024-12-02T21:17:09,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/d0af24489a1042058defe046ad7f1d73 is 50, key is test_row_0/C:col10/1733174226004/Put/seqid=0 2024-12-02T21:17:09,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742236_1412 (size=12301) 2024-12-02T21:17:09,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/d0af24489a1042058defe046ad7f1d73 2024-12-02T21:17:09,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/dad3c2167a6f485198916a370412a0a8 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dad3c2167a6f485198916a370412a0a8 2024-12-02T21:17:09,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dad3c2167a6f485198916a370412a0a8, entries=150, sequenceid=368, filesize=30.5 K 2024-12-02T21:17:09,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/a67de0a3e03f4e49b1c8a2c77b2b5c7c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/a67de0a3e03f4e49b1c8a2c77b2b5c7c 2024-12-02T21:17:09,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/a67de0a3e03f4e49b1c8a2c77b2b5c7c, entries=150, sequenceid=368, filesize=12.0 K 2024-12-02T21:17:09,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/d0af24489a1042058defe046ad7f1d73 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/d0af24489a1042058defe046ad7f1d73 2024-12-02T21:17:09,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/d0af24489a1042058defe046ad7f1d73, entries=150, sequenceid=368, filesize=12.0 K 2024-12-02T21:17:09,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=6.71 KB/6870 for a3e315252bf6ea46b1e790171c7a96a1 in 1667ms, sequenceid=368, compaction requested=false 2024-12-02T21:17:09,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e315252bf6ea46b1e790171c7a96a1: 2024-12-02T21:17:12,179 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T21:17:15,062 INFO [Thread-1459 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1459. 2024-12-02T21:17:15,062 DEBUG [Thread-1459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:15,062 DEBUG [Thread-1459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:15,062 DEBUG [Thread-1459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:15,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:15,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T21:17:15,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-12-02T21:17:15,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1456 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4367 rows 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1460 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4376 rows 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1457 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4370 rows 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1461 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4382 rows 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1467 2024-12-02T21:17:15,063 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4391 rows 2024-12-02T21:17:15,063 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:17:15,063 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:17:15,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:15,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:15,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:15,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T21:17:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T21:17:15,070 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174235070"}]},"ts":"1733174235070"} 2024-12-02T21:17:15,071 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T21:17:15,071 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T21:17:15,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T21:17:15,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, UNASSIGN}] 2024-12-02T21:17:15,074 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, UNASSIGN 2024-12-02T21:17:15,075 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=a3e315252bf6ea46b1e790171c7a96a1, regionState=CLOSING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:15,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, UNASSIGN because future has completed 2024-12-02T21:17:15,076 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:17:15,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:17:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T21:17:15,229 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:15,229 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:17:15,230 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing a3e315252bf6ea46b1e790171c7a96a1, disabling compactions & flushes 2024-12-02T21:17:15,230 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:15,230 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:15,230 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. after waiting 0 ms 2024-12-02T21:17:15,230 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:15,230 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing a3e315252bf6ea46b1e790171c7a96a1 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-02T21:17:15,231 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=A 2024-12-02T21:17:15,231 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:15,231 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=B 2024-12-02T21:17:15,231 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:15,231 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e315252bf6ea46b1e790171c7a96a1, store=C 2024-12-02T21:17:15,231 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:15,239 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022c9360db1dfa4c30863dccb204859794_a3e315252bf6ea46b1e790171c7a96a1 is 50, key is test_row_0/A:col10/1733174235059/Put/seqid=0 2024-12-02T21:17:15,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742237_1413 (size=7374) 2024-12-02T21:17:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T21:17:15,644 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:15,653 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022c9360db1dfa4c30863dccb204859794_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022c9360db1dfa4c30863dccb204859794_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:15,654 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/4b67e78940fa490e98a159443698a626, store: [table=TestAcidGuarantees family=A region=a3e315252bf6ea46b1e790171c7a96a1] 2024-12-02T21:17:15,655 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/4b67e78940fa490e98a159443698a626 is 175, key is test_row_0/A:col10/1733174235059/Put/seqid=0 2024-12-02T21:17:15,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742238_1414 (size=13865) 2024-12-02T21:17:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T21:17:15,938 WARN [regionserver/87c3fdb6c570:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-02T21:17:15,939 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/6f92526a1b344447938c558e80d8b01a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/448f4f63c4194a35893b92c1eac199ef, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dbbf5ef162aa4e0b8a766969c22f52fd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9ecb234774e94c04b6e5250f1e268cc4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/85bda578c27f476fa0ae81266648ae8e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/e585c3ff6c8741bdba1efd18c2b5060b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/1252ea2334c444478ed53e1ceeffe848, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/f65440cd32c245d58dc0d4978bb1f689, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a09af92fe4394a19a980e1ab45b1d445, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/fe1404c0abdc45f3838a0ea0e70eddd3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/80df58623dc34049ae051dec0cf333fe, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/5b55062e1ed84fd19ab2b55afbee59c4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/8907fadc973a4c9f9f925cb50346e7b5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/bd51df34db094a7491fe1066b5e03296, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/29ad91e212224c0488208be35f4c5023, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/318410b604d64455b2fc39fe87eeac6b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/59182562e9104a5789d0c9c8f5e05d7d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7d7d0feeda514815a1e0fd2ba82865ef, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7e4d2606c5164ec3abb90e88e5a23793, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/eef09f1aa9344fbfb6471d7157cdd6ad] to archive 2024-12-02T21:17:15,940 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:17:15,942 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/6f92526a1b344447938c558e80d8b01a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/6f92526a1b344447938c558e80d8b01a 2024-12-02T21:17:15,943 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9d2c15eaf06e4ba1b6e6b06d6eb5556d 2024-12-02T21:17:15,944 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/448f4f63c4194a35893b92c1eac199ef to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/448f4f63c4194a35893b92c1eac199ef 2024-12-02T21:17:15,945 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dbbf5ef162aa4e0b8a766969c22f52fd to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dbbf5ef162aa4e0b8a766969c22f52fd 2024-12-02T21:17:15,946 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9ecb234774e94c04b6e5250f1e268cc4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/9ecb234774e94c04b6e5250f1e268cc4 2024-12-02T21:17:15,946 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/85bda578c27f476fa0ae81266648ae8e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/85bda578c27f476fa0ae81266648ae8e 2024-12-02T21:17:15,947 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/e585c3ff6c8741bdba1efd18c2b5060b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/e585c3ff6c8741bdba1efd18c2b5060b 2024-12-02T21:17:15,948 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6b1ce9f2e9f4d4fad2bcd3e2dc7d5aa 2024-12-02T21:17:15,949 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/1252ea2334c444478ed53e1ceeffe848 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/1252ea2334c444478ed53e1ceeffe848 2024-12-02T21:17:15,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/f65440cd32c245d58dc0d4978bb1f689 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/f65440cd32c245d58dc0d4978bb1f689 2024-12-02T21:17:15,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a09af92fe4394a19a980e1ab45b1d445 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a09af92fe4394a19a980e1ab45b1d445 2024-12-02T21:17:15,951 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/fe1404c0abdc45f3838a0ea0e70eddd3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/fe1404c0abdc45f3838a0ea0e70eddd3 2024-12-02T21:17:15,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/80df58623dc34049ae051dec0cf333fe to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/80df58623dc34049ae051dec0cf333fe 2024-12-02T21:17:15,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/5b55062e1ed84fd19ab2b55afbee59c4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/5b55062e1ed84fd19ab2b55afbee59c4 2024-12-02T21:17:15,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/8907fadc973a4c9f9f925cb50346e7b5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/8907fadc973a4c9f9f925cb50346e7b5 2024-12-02T21:17:15,954 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/bd51df34db094a7491fe1066b5e03296 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/bd51df34db094a7491fe1066b5e03296 2024-12-02T21:17:15,955 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/29ad91e212224c0488208be35f4c5023 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/29ad91e212224c0488208be35f4c5023 2024-12-02T21:17:15,956 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/318410b604d64455b2fc39fe87eeac6b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/318410b604d64455b2fc39fe87eeac6b 2024-12-02T21:17:15,956 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/59182562e9104a5789d0c9c8f5e05d7d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/59182562e9104a5789d0c9c8f5e05d7d 2024-12-02T21:17:15,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7d7d0feeda514815a1e0fd2ba82865ef to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7d7d0feeda514815a1e0fd2ba82865ef 2024-12-02T21:17:15,959 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7e4d2606c5164ec3abb90e88e5a23793 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/7e4d2606c5164ec3abb90e88e5a23793 2024-12-02T21:17:15,959 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/eef09f1aa9344fbfb6471d7157cdd6ad to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/eef09f1aa9344fbfb6471d7157cdd6ad 2024-12-02T21:17:15,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/8d1046b39e2f4edfa1476071af1d8757, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/f4082892ac0c46b58188854dff262bcd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/a46dd1d942c64afa8642dec4d554fc8b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9ebdbfc68da44eedaec4b2fdb3535f35, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4fcda08de5cb44c68c015369fcde2a7a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/cf11be157225497aa2ee871234c0f8ee, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/6f4d3aec08c0416b8a35a8eeafe8fd5f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/5ee13ecf12a04b9d87dd959505fe9a7b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ab07fcea69854f4a8c53ece92c84aa5d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9faec3c572cc49cd8b7ce8f3c76cf147, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/27e1d389393d4a73b9a257d946d660aa, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e9fd76f360474fe9a9091264807c81bf, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ea440679696942c98bdb56a30abf124f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ffdfc7b5a20d4e72a302d254eab208a4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed965f20d4bf434e8c1d9dfe0cf1741d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed0ae257884c49d9909bd9d934c46cf4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4c1625093ca143618d390f35150eb834, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e31fba67c1d2484886b4b932c5e192df, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/3da80309054043ab8a8c361e4287ab04, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/0643fe652be54823a33e552d655e5d3d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/7b7ea6f25dd14062acc1fa723cb8828d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e6b32f2b21b14595bea6143ecaebd36a] to archive 2024-12-02T21:17:15,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:17:15,966 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/8d1046b39e2f4edfa1476071af1d8757 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/8d1046b39e2f4edfa1476071af1d8757 2024-12-02T21:17:15,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/f4082892ac0c46b58188854dff262bcd to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/f4082892ac0c46b58188854dff262bcd 2024-12-02T21:17:15,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/a46dd1d942c64afa8642dec4d554fc8b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/a46dd1d942c64afa8642dec4d554fc8b 2024-12-02T21:17:15,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9ebdbfc68da44eedaec4b2fdb3535f35 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9ebdbfc68da44eedaec4b2fdb3535f35 2024-12-02T21:17:15,970 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4fcda08de5cb44c68c015369fcde2a7a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4fcda08de5cb44c68c015369fcde2a7a 2024-12-02T21:17:15,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/cf11be157225497aa2ee871234c0f8ee to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/cf11be157225497aa2ee871234c0f8ee 2024-12-02T21:17:15,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/6f4d3aec08c0416b8a35a8eeafe8fd5f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/6f4d3aec08c0416b8a35a8eeafe8fd5f 2024-12-02T21:17:15,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/5ee13ecf12a04b9d87dd959505fe9a7b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/5ee13ecf12a04b9d87dd959505fe9a7b 2024-12-02T21:17:15,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ab07fcea69854f4a8c53ece92c84aa5d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ab07fcea69854f4a8c53ece92c84aa5d 2024-12-02T21:17:15,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9faec3c572cc49cd8b7ce8f3c76cf147 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/9faec3c572cc49cd8b7ce8f3c76cf147 2024-12-02T21:17:15,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/27e1d389393d4a73b9a257d946d660aa to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/27e1d389393d4a73b9a257d946d660aa 2024-12-02T21:17:15,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e9fd76f360474fe9a9091264807c81bf to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e9fd76f360474fe9a9091264807c81bf 2024-12-02T21:17:15,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ea440679696942c98bdb56a30abf124f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ea440679696942c98bdb56a30abf124f 2024-12-02T21:17:15,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ffdfc7b5a20d4e72a302d254eab208a4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ffdfc7b5a20d4e72a302d254eab208a4 2024-12-02T21:17:15,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed965f20d4bf434e8c1d9dfe0cf1741d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed965f20d4bf434e8c1d9dfe0cf1741d 2024-12-02T21:17:15,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed0ae257884c49d9909bd9d934c46cf4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/ed0ae257884c49d9909bd9d934c46cf4 2024-12-02T21:17:15,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4c1625093ca143618d390f35150eb834 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/4c1625093ca143618d390f35150eb834 2024-12-02T21:17:15,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e31fba67c1d2484886b4b932c5e192df to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e31fba67c1d2484886b4b932c5e192df 2024-12-02T21:17:15,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/3da80309054043ab8a8c361e4287ab04 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/3da80309054043ab8a8c361e4287ab04 2024-12-02T21:17:15,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/0643fe652be54823a33e552d655e5d3d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/0643fe652be54823a33e552d655e5d3d 2024-12-02T21:17:15,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/7b7ea6f25dd14062acc1fa723cb8828d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/7b7ea6f25dd14062acc1fa723cb8828d 2024-12-02T21:17:15,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e6b32f2b21b14595bea6143ecaebd36a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/e6b32f2b21b14595bea6143ecaebd36a 2024-12-02T21:17:15,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/110f671141d94643bdaac2de3f3bea93, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/a698d1d30bf74e01b9fc39440ff0db49, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/20f52c84ddb6440e9216d64eb866aef9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/86dee15f75144f5584b6017901c2fdf8, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/02b68b26985b4dbeb73075e745e2527e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/cd681b8255f843809d4680655362281a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/83dc407583544119a6c6b352c05025e6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/351cf40c88be482792fa8614a65225b9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6db97dd1cbdd4928abae95f4c41fce95, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/7ca03bbab9054c2298d0a7a9eb6442d9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/56135e8c8b344abcab79c6014ebf2fd6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/db3c4390064e4bd38837bfcca1b34999, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/9190cf766bbb4f938382878bcac5f57d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/778fa0ed240e458c9e300a51d38bff9d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/223c367a63334a4886a093c70bffc91a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/1957b55c7cec45b59652d87031ec2f91, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/aa71309ed3ce40debaf1867fd8184eba, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/2854f292164f4b92809392d74c0915b1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/33dc8f9b3cae4365952d35e3c00605fa, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/189f39fcf2614db39e3c9d9a193f5195, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/3bd0c2c3daf74189a04a7a5ccedb8ae9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6c7b766c4d3c4b19aa8f852188b163b4] to archive 2024-12-02T21:17:15,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:17:15,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/110f671141d94643bdaac2de3f3bea93 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/110f671141d94643bdaac2de3f3bea93 2024-12-02T21:17:15,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/a698d1d30bf74e01b9fc39440ff0db49 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/a698d1d30bf74e01b9fc39440ff0db49 2024-12-02T21:17:15,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/20f52c84ddb6440e9216d64eb866aef9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/20f52c84ddb6440e9216d64eb866aef9 2024-12-02T21:17:15,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/86dee15f75144f5584b6017901c2fdf8 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/86dee15f75144f5584b6017901c2fdf8 2024-12-02T21:17:15,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/02b68b26985b4dbeb73075e745e2527e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/02b68b26985b4dbeb73075e745e2527e 2024-12-02T21:17:15,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/cd681b8255f843809d4680655362281a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/cd681b8255f843809d4680655362281a 2024-12-02T21:17:15,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/83dc407583544119a6c6b352c05025e6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/83dc407583544119a6c6b352c05025e6 2024-12-02T21:17:15,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/351cf40c88be482792fa8614a65225b9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/351cf40c88be482792fa8614a65225b9 2024-12-02T21:17:15,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6db97dd1cbdd4928abae95f4c41fce95 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6db97dd1cbdd4928abae95f4c41fce95 2024-12-02T21:17:15,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/7ca03bbab9054c2298d0a7a9eb6442d9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/7ca03bbab9054c2298d0a7a9eb6442d9 2024-12-02T21:17:15,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/56135e8c8b344abcab79c6014ebf2fd6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/56135e8c8b344abcab79c6014ebf2fd6 2024-12-02T21:17:15,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/db3c4390064e4bd38837bfcca1b34999 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/db3c4390064e4bd38837bfcca1b34999 2024-12-02T21:17:15,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/9190cf766bbb4f938382878bcac5f57d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/9190cf766bbb4f938382878bcac5f57d 2024-12-02T21:17:15,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/778fa0ed240e458c9e300a51d38bff9d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/778fa0ed240e458c9e300a51d38bff9d 2024-12-02T21:17:15,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/223c367a63334a4886a093c70bffc91a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/223c367a63334a4886a093c70bffc91a 2024-12-02T21:17:15,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/1957b55c7cec45b59652d87031ec2f91 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/1957b55c7cec45b59652d87031ec2f91 2024-12-02T21:17:15,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/aa71309ed3ce40debaf1867fd8184eba to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/aa71309ed3ce40debaf1867fd8184eba 2024-12-02T21:17:16,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/2854f292164f4b92809392d74c0915b1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/2854f292164f4b92809392d74c0915b1 2024-12-02T21:17:16,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/33dc8f9b3cae4365952d35e3c00605fa to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/33dc8f9b3cae4365952d35e3c00605fa 2024-12-02T21:17:16,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/189f39fcf2614db39e3c9d9a193f5195 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/189f39fcf2614db39e3c9d9a193f5195 2024-12-02T21:17:16,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/3bd0c2c3daf74189a04a7a5ccedb8ae9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/3bd0c2c3daf74189a04a7a5ccedb8ae9 2024-12-02T21:17:16,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6c7b766c4d3c4b19aa8f852188b163b4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/6c7b766c4d3c4b19aa8f852188b163b4 2024-12-02T21:17:16,061 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/4b67e78940fa490e98a159443698a626 2024-12-02T21:17:16,066 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/2bd029d41b6b4887913d61ee0cf79f9f is 50, key is test_row_0/B:col10/1733174235059/Put/seqid=0 2024-12-02T21:17:16,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742239_1415 (size=7415) 2024-12-02T21:17:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T21:17:16,469 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/2bd029d41b6b4887913d61ee0cf79f9f 2024-12-02T21:17:16,476 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/b1f8b4db9c004ffcabf39fb006a5c52b is 50, key is test_row_0/C:col10/1733174235059/Put/seqid=0 2024-12-02T21:17:16,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742240_1416 (size=7415) 2024-12-02T21:17:16,880 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/b1f8b4db9c004ffcabf39fb006a5c52b 2024-12-02T21:17:16,884 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/A/4b67e78940fa490e98a159443698a626 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/4b67e78940fa490e98a159443698a626 2024-12-02T21:17:16,887 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/4b67e78940fa490e98a159443698a626, entries=50, sequenceid=374, filesize=13.5 K 2024-12-02T21:17:16,888 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/B/2bd029d41b6b4887913d61ee0cf79f9f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/2bd029d41b6b4887913d61ee0cf79f9f 2024-12-02T21:17:16,898 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/2bd029d41b6b4887913d61ee0cf79f9f, entries=50, sequenceid=374, filesize=7.2 K 2024-12-02T21:17:16,899 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/.tmp/C/b1f8b4db9c004ffcabf39fb006a5c52b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/b1f8b4db9c004ffcabf39fb006a5c52b 2024-12-02T21:17:16,903 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/b1f8b4db9c004ffcabf39fb006a5c52b, entries=50, sequenceid=374, filesize=7.2 K 2024-12-02T21:17:16,904 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for a3e315252bf6ea46b1e790171c7a96a1 in 1674ms, sequenceid=374, compaction requested=true 2024-12-02T21:17:16,913 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/recovered.edits/377.seqid, newMaxSeqId=377, maxSeqId=4 2024-12-02T21:17:16,913 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. 2024-12-02T21:17:16,913 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for a3e315252bf6ea46b1e790171c7a96a1: Waiting for close lock at 1733174235230Running coprocessor pre-close hooks at 1733174235230Disabling compacts and flushes for region at 1733174235230Disabling writes for close at 1733174235230Obtaining lock to block concurrent updates at 1733174235230Preparing flush snapshotting stores in a3e315252bf6ea46b1e790171c7a96a1 at 1733174235230Finished memstore snapshotting TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733174235231 (+1 ms)Flushing stores of TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1. at 1733174235233 (+2 ms)Flushing a3e315252bf6ea46b1e790171c7a96a1/A: creating writer at 1733174235233Flushing mob file a3e315252bf6ea46b1e790171c7a96a1/A: appending metadata at 1733174235239 (+6 ms)Flushing mob file a3e315252bf6ea46b1e790171c7a96a1/A: closing flushed file at 1733174235239Flushing a3e315252bf6ea46b1e790171c7a96a1/A: appending metadata at 1733174235654 (+415 ms)Flushing a3e315252bf6ea46b1e790171c7a96a1/A: closing flushed file at 1733174235655 (+1 ms)Flushing a3e315252bf6ea46b1e790171c7a96a1/B: creating writer at 1733174236064 (+409 ms)Flushing a3e315252bf6ea46b1e790171c7a96a1/B: appending metadata at 1733174236065 (+1 ms)Flushing a3e315252bf6ea46b1e790171c7a96a1/B: closing flushed file at 1733174236065Flushing a3e315252bf6ea46b1e790171c7a96a1/C: creating writer at 1733174236473 (+408 ms)Flushing a3e315252bf6ea46b1e790171c7a96a1/C: appending metadata at 1733174236475 (+2 ms)Flushing a3e315252bf6ea46b1e790171c7a96a1/C: closing flushed file at 1733174236475Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@128302a6: reopening flushed file at 1733174236883 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5102eb63: reopening flushed file at 1733174236887 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38e5ee43: reopening flushed file at 1733174236898 (+11 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for a3e315252bf6ea46b1e790171c7a96a1 in 1674ms, sequenceid=374, compaction requested=true at 1733174236904 (+6 ms)Writing region close event to WAL at 1733174236905 (+1 ms)Running coprocessor post-close hooks at 1733174236913 (+8 ms)Closed at 1733174236913 2024-12-02T21:17:16,915 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:16,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=a3e315252bf6ea46b1e790171c7a96a1, regionState=CLOSED 2024-12-02T21:17:16,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:17:16,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-02T21:17:16,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure a3e315252bf6ea46b1e790171c7a96a1, server=87c3fdb6c570,35563,1733174113859 in 1.8420 sec 2024-12-02T21:17:16,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-02T21:17:16,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e315252bf6ea46b1e790171c7a96a1, UNASSIGN in 1.8460 sec 2024-12-02T21:17:16,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-02T21:17:16,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8490 sec 2024-12-02T21:17:16,923 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174236923"}]},"ts":"1733174236923"} 2024-12-02T21:17:16,924 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T21:17:16,924 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T21:17:16,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8600 sec 2024-12-02T21:17:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T21:17:17,220 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:17,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T21:17:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:17,224 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T21:17:17,226 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:17,229 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,232 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/recovered.edits] 2024-12-02T21:17:17,235 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/4b67e78940fa490e98a159443698a626 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/4b67e78940fa490e98a159443698a626 2024-12-02T21:17:17,236 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6294b1f9b8140a1bdb76563fad7b503 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/a6294b1f9b8140a1bdb76563fad7b503 2024-12-02T21:17:17,237 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dad3c2167a6f485198916a370412a0a8 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/A/dad3c2167a6f485198916a370412a0a8 2024-12-02T21:17:17,239 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/2bd029d41b6b4887913d61ee0cf79f9f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/2bd029d41b6b4887913d61ee0cf79f9f 2024-12-02T21:17:17,240 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/80feffba9c944b64b4fc531de2b7082c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/80feffba9c944b64b4fc531de2b7082c 2024-12-02T21:17:17,241 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/a67de0a3e03f4e49b1c8a2c77b2b5c7c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/B/a67de0a3e03f4e49b1c8a2c77b2b5c7c 2024-12-02T21:17:17,243 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/aa10d757349b44f9ba858465afe39e99 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/aa10d757349b44f9ba858465afe39e99 2024-12-02T21:17:17,244 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/b1f8b4db9c004ffcabf39fb006a5c52b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/b1f8b4db9c004ffcabf39fb006a5c52b 2024-12-02T21:17:17,245 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/d0af24489a1042058defe046ad7f1d73 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/C/d0af24489a1042058defe046ad7f1d73 2024-12-02T21:17:17,248 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/recovered.edits/377.seqid to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1/recovered.edits/377.seqid 2024-12-02T21:17:17,249 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,249 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T21:17:17,249 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T21:17:17,250 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T21:17:17,253 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020a0ba7cc278840878071fd20df980410_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020a0ba7cc278840878071fd20df980410_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,254 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120223827a5167f04d6ab1759c27b723606a_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120223827a5167f04d6ab1759c27b723606a_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,255 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022497c81eeadc47fbb70f2bb2e7e24cb2_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022497c81eeadc47fbb70f2bb2e7e24cb2_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,255 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022c9360db1dfa4c30863dccb204859794_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022c9360db1dfa4c30863dccb204859794_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,256 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023cd5bf9527634cfaba63986397a20592_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023cd5bf9527634cfaba63986397a20592_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,257 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120240d188302f20428497477ef417c8695b_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120240d188302f20428497477ef417c8695b_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,258 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024309f699d79847b09ca38f0f6f8c72d6_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024309f699d79847b09ca38f0f6f8c72d6_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,258 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120244d197e042344f5a9d981e14fae28cca_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120244d197e042344f5a9d981e14fae28cca_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,259 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025c73ea40a6eb48ceb62f340ecaf7bf12_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025c73ea40a6eb48ceb62f340ecaf7bf12_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,260 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202604c920d857c429db4b981454c880661_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202604c920d857c429db4b981454c880661_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,260 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026301e21cf10948758b64517e41d17afc_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026301e21cf10948758b64517e41d17afc_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,261 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026d1cf2541cbe48709108c49173e5e2dd_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026d1cf2541cbe48709108c49173e5e2dd_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,261 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027ea83612b6ee42058b355268bde8ecf8_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027ea83612b6ee42058b355268bde8ecf8_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,262 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028bd45894b915416a925758a5d35a95a8_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028bd45894b915416a925758a5d35a95a8_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,263 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028dcc1dbf85414c2b9611ceee19fe62eb_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028dcc1dbf85414c2b9611ceee19fe62eb_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,264 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029a5eb261be8d414c87abae568e9aa86e_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029a5eb261be8d414c87abae568e9aa86e_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,265 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b8adb12502604d4dbc981f83889ba4aa_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b8adb12502604d4dbc981f83889ba4aa_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,266 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ed89441a183e42a4b91a7e3f08d23dcc_a3e315252bf6ea46b1e790171c7a96a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ed89441a183e42a4b91a7e3f08d23dcc_a3e315252bf6ea46b1e790171c7a96a1 2024-12-02T21:17:17,266 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T21:17:17,269 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:17,271 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T21:17:17,273 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T21:17:17,274 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:17,274 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T21:17:17,274 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733174237274"}]},"ts":"9223372036854775807"} 2024-12-02T21:17:17,276 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T21:17:17,276 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a3e315252bf6ea46b1e790171c7a96a1, NAME => 'TestAcidGuarantees,,1733174203375.a3e315252bf6ea46b1e790171c7a96a1.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T21:17:17,277 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T21:17:17,277 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733174237277"}]},"ts":"9223372036854775807"} 2024-12-02T21:17:17,278 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T21:17:17,279 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:17,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-12-02T21:17:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T21:17:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T21:17:17,334 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:17,345 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=462 (was 436) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2396 (was 1902) - AvailableMemoryMB LEAK? - 2024-12-02T21:17:17,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=462, ProcessCount=11, AvailableMemoryMB=2395 2024-12-02T21:17:17,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T21:17:17,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:17:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:17,357 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T21:17:17,358 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:17,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-12-02T21:17:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-02T21:17:17,358 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T21:17:17,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742241_1417 (size=1061) 2024-12-02T21:17:17,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-02T21:17:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-02T21:17:17,770 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0 2024-12-02T21:17:17,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742242_1418 (size=53) 2024-12-02T21:17:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-02T21:17:18,177 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:17:18,177 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 62c2292167ae6a525d2d66119fcc23eb, disabling compactions & flushes 2024-12-02T21:17:18,177 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:18,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:18,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. after waiting 0 ms 2024-12-02T21:17:18,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:18,178 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:18,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 62c2292167ae6a525d2d66119fcc23eb: Waiting for close lock at 1733174238177Disabling compacts and flushes for region at 1733174238177Disabling writes for close at 1733174238178 (+1 ms)Writing region close event to WAL at 1733174238178Closed at 1733174238178 2024-12-02T21:17:18,180 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T21:17:18,180 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733174238180"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174238180"}]},"ts":"1733174238180"} 2024-12-02T21:17:18,183 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T21:17:18,185 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T21:17:18,186 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174238185"}]},"ts":"1733174238185"} 2024-12-02T21:17:18,188 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T21:17:18,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62c2292167ae6a525d2d66119fcc23eb, ASSIGN}] 2024-12-02T21:17:18,189 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62c2292167ae6a525d2d66119fcc23eb, ASSIGN 2024-12-02T21:17:18,190 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62c2292167ae6a525d2d66119fcc23eb, ASSIGN; state=OFFLINE, location=87c3fdb6c570,35563,1733174113859; forceNewPlan=false, retain=false 2024-12-02T21:17:18,342 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=62c2292167ae6a525d2d66119fcc23eb, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:18,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62c2292167ae6a525d2d66119fcc23eb, ASSIGN because future has completed 2024-12-02T21:17:18,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:17:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-02T21:17:18,505 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:18,506 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:17:18,507 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,507 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:17:18,507 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,507 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,508 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,510 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:17:18,510 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62c2292167ae6a525d2d66119fcc23eb columnFamilyName A 2024-12-02T21:17:18,510 DEBUG [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:18,511 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] regionserver.HStore(327): Store=62c2292167ae6a525d2d66119fcc23eb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:17:18,511 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,512 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:17:18,513 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62c2292167ae6a525d2d66119fcc23eb columnFamilyName B 2024-12-02T21:17:18,513 DEBUG [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:18,513 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] regionserver.HStore(327): Store=62c2292167ae6a525d2d66119fcc23eb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:17:18,513 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,514 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:17:18,514 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62c2292167ae6a525d2d66119fcc23eb columnFamilyName C 2024-12-02T21:17:18,514 DEBUG [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:18,514 INFO [StoreOpener-62c2292167ae6a525d2d66119fcc23eb-1 {}] regionserver.HStore(327): Store=62c2292167ae6a525d2d66119fcc23eb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:17:18,514 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:18,515 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,515 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,515 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,516 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,516 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,516 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:17:18,517 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,519 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:17:18,519 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 62c2292167ae6a525d2d66119fcc23eb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70827910, jitterRate=0.05541810393333435}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:17:18,519 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:18,520 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 62c2292167ae6a525d2d66119fcc23eb: Running coprocessor pre-open hook at 1733174238507Writing region info on filesystem at 1733174238507Initializing all the Stores at 1733174238508 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174238508Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174238508Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174238508Cleaning up temporary data from old regions at 1733174238516 (+8 ms)Running coprocessor post-open hooks at 1733174238519 (+3 ms)Region opened successfully at 1733174238520 (+1 ms) 2024-12-02T21:17:18,520 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., pid=116, masterSystemTime=1733174238500 2024-12-02T21:17:18,522 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:18,522 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:18,522 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=62c2292167ae6a525d2d66119fcc23eb, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:18,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:17:18,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-02T21:17:18,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 in 178 msec 2024-12-02T21:17:18,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-02T21:17:18,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62c2292167ae6a525d2d66119fcc23eb, ASSIGN in 338 msec 2024-12-02T21:17:18,528 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T21:17:18,528 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174238528"}]},"ts":"1733174238528"} 2024-12-02T21:17:18,529 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T21:17:18,530 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T21:17:18,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-12-02T21:17:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-02T21:17:19,520 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:19,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,524 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,527 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223b206f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2448f177, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,532 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,534 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T21:17:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 87c3fdb6c570,39577,1733174113031 2024-12-02T21:17:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6dc3679f 2024-12-02T21:17:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T21:17:19,536 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T21:17:19,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,537 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600c5898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,539 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@609b3570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,540 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e58cc63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,541 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b56e352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,543 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9911ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,544 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559a7138, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,545 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b20eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,546 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633d63c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a851dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f6dbfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,550 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79e73927, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,552 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e0c629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b52077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,555 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16a01a7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2a9add, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11744533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c68dbb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,560 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134ee4d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:19,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:19,561 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41251067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:19,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:19,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1adf50a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:19,563 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,563 DEBUG [Thread-1892 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,564 DEBUG [Thread-1896 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,564 DEBUG [Thread-1907 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,564 DEBUG [Thread-1905 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,565 DEBUG [Thread-1890 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,565 DEBUG [Thread-1888 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,566 DEBUG [Thread-1894 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,567 DEBUG [Thread-1899 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,567 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-02T21:17:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,568 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:19,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:19,570 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T21:17:19,571 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:19,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:19,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2] 2024-12-02T21:17:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:19,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:17:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:19,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/09f4ffe7067c4d3ebee52b2c69f3c868 is 50, key is test_row_0/A:col10/1733174239577/Put/seqid=0 2024-12-02T21:17:19,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174299603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742243_1419 (size=14341) 2024-12-02T21:17:19,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/09f4ffe7067c4d3ebee52b2c69f3c868 2024-12-02T21:17:19,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174299609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174299612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174299613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174299615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/8640b2ca41194b4987df887be2532ab6 is 50, key is test_row_0/B:col10/1733174239577/Put/seqid=0 2024-12-02T21:17:19,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742244_1420 (size=12001) 2024-12-02T21:17:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T21:17:19,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174299718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T21:17:19,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:19,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:19,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174299729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174299729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174299729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174299729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T21:17:19,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:19,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:19,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:19,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:19,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T21:17:19,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174299928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174299943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174299943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174299943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:19,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174299943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T21:17:20,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:20,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:20,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:20,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:20,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:20,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/8640b2ca41194b4987df887be2532ab6 2024-12-02T21:17:20,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/37773f69f41a47d0a545fb1d49c388ff is 50, key is test_row_0/C:col10/1733174239577/Put/seqid=0 2024-12-02T21:17:20,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742245_1421 (size=12001) 2024-12-02T21:17:20,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T21:17:20,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:20,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:20,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T21:17:20,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174300239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174300259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174300259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174300259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174300259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T21:17:20,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:20,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:20,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:20,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:20,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:20,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/37773f69f41a47d0a545fb1d49c388ff 2024-12-02T21:17:20,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/09f4ffe7067c4d3ebee52b2c69f3c868 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/09f4ffe7067c4d3ebee52b2c69f3c868 2024-12-02T21:17:20,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/09f4ffe7067c4d3ebee52b2c69f3c868, entries=200, sequenceid=13, filesize=14.0 K 2024-12-02T21:17:20,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/8640b2ca41194b4987df887be2532ab6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8640b2ca41194b4987df887be2532ab6 2024-12-02T21:17:20,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8640b2ca41194b4987df887be2532ab6, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T21:17:20,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/37773f69f41a47d0a545fb1d49c388ff as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/37773f69f41a47d0a545fb1d49c388ff 2024-12-02T21:17:20,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/37773f69f41a47d0a545fb1d49c388ff, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T21:17:20,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 62c2292167ae6a525d2d66119fcc23eb in 900ms, sequenceid=13, compaction requested=false 2024-12-02T21:17:20,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:20,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T21:17:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:20,488 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T21:17:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/c06c79ace1e947188602e00c4edbd3bd is 50, key is test_row_0/A:col10/1733174239613/Put/seqid=0 2024-12-02T21:17:20,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742246_1422 (size=12001) 2024-12-02T21:17:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T21:17:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:20,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:20,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174300774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174300775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174300776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174300778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174300779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174300889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174300889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174300889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174300889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:20,910 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/c06c79ace1e947188602e00c4edbd3bd 2024-12-02T21:17:20,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/9137fbe726e141b5ba84f2ffaffc271c is 50, key is test_row_0/B:col10/1733174239613/Put/seqid=0 2024-12-02T21:17:20,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742247_1423 (size=12001) 2024-12-02T21:17:21,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174301099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174301099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174301099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174301099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,321 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/9137fbe726e141b5ba84f2ffaffc271c 2024-12-02T21:17:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/c5ad07b87d4747899e593cb1e399b1e4 is 50, key is test_row_0/C:col10/1733174239613/Put/seqid=0 2024-12-02T21:17:21,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742248_1424 (size=12001) 2024-12-02T21:17:21,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174301408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174301408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174301408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174301408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T21:17:21,730 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/c5ad07b87d4747899e593cb1e399b1e4 2024-12-02T21:17:21,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/c06c79ace1e947188602e00c4edbd3bd as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/c06c79ace1e947188602e00c4edbd3bd 2024-12-02T21:17:21,739 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/c06c79ace1e947188602e00c4edbd3bd, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T21:17:21,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/9137fbe726e141b5ba84f2ffaffc271c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/9137fbe726e141b5ba84f2ffaffc271c 2024-12-02T21:17:21,744 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/9137fbe726e141b5ba84f2ffaffc271c, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T21:17:21,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/c5ad07b87d4747899e593cb1e399b1e4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c5ad07b87d4747899e593cb1e399b1e4 2024-12-02T21:17:21,751 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c5ad07b87d4747899e593cb1e399b1e4, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T21:17:21,752 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 62c2292167ae6a525d2d66119fcc23eb in 1264ms, sequenceid=33, compaction requested=false 2024-12-02T21:17:21,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:21,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:21,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-02T21:17:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-02T21:17:21,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-02T21:17:21,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1830 sec 2024-12-02T21:17:21,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.1930 sec 2024-12-02T21:17:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:21,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T21:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:21,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/e8f9809d1cee48b5bfb1c68e43fe4c57 is 50, key is test_row_0/A:col10/1733174241799/Put/seqid=0 2024-12-02T21:17:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742249_1425 (size=14341) 2024-12-02T21:17:21,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174301909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174301919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174301929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174301929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174301929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:22,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174302019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:22,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/e8f9809d1cee48b5bfb1c68e43fe4c57 2024-12-02T21:17:22,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/6d465b3f1ad34c2f8de254cf2e05c427 is 50, key is test_row_0/B:col10/1733174241799/Put/seqid=0 2024-12-02T21:17:22,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174302229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:22,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742250_1426 (size=12001) 2024-12-02T21:17:22,434 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T21:17:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174302538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:22,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/6d465b3f1ad34c2f8de254cf2e05c427 2024-12-02T21:17:22,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/6352fa1292b64d598544d97d97b99aa0 is 50, key is test_row_0/C:col10/1733174241799/Put/seqid=0 2024-12-02T21:17:22,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742251_1427 (size=12001) 2024-12-02T21:17:22,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174302939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:22,945 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:22,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:22,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174302948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:22,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174302949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:22,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174302949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:22,953 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,954 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:22,954 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:23,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/6352fa1292b64d598544d97d97b99aa0 2024-12-02T21:17:23,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/e8f9809d1cee48b5bfb1c68e43fe4c57 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/e8f9809d1cee48b5bfb1c68e43fe4c57 2024-12-02T21:17:23,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174303059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:23,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/e8f9809d1cee48b5bfb1c68e43fe4c57, entries=200, sequenceid=46, filesize=14.0 K 2024-12-02T21:17:23,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/6d465b3f1ad34c2f8de254cf2e05c427 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/6d465b3f1ad34c2f8de254cf2e05c427 2024-12-02T21:17:23,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/6d465b3f1ad34c2f8de254cf2e05c427, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T21:17:23,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/6352fa1292b64d598544d97d97b99aa0 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/6352fa1292b64d598544d97d97b99aa0 2024-12-02T21:17:23,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/6352fa1292b64d598544d97d97b99aa0, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T21:17:23,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 62c2292167ae6a525d2d66119fcc23eb in 1295ms, sequenceid=46, compaction requested=true 2024-12-02T21:17:23,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:23,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:23,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:23,096 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:23,096 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:23,096 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:23,096 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:23,097 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/B is initiating minor compaction (all files) 2024-12-02T21:17:23,097 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/A is initiating minor compaction (all files) 2024-12-02T21:17:23,097 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/A in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:23,097 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/B in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:23,097 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8640b2ca41194b4987df887be2532ab6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/9137fbe726e141b5ba84f2ffaffc271c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/6d465b3f1ad34c2f8de254cf2e05c427] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=35.2 K 2024-12-02T21:17:23,097 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/09f4ffe7067c4d3ebee52b2c69f3c868, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/c06c79ace1e947188602e00c4edbd3bd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/e8f9809d1cee48b5bfb1c68e43fe4c57] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=39.7 K 2024-12-02T21:17:23,100 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 8640b2ca41194b4987df887be2532ab6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733174239577 2024-12-02T21:17:23,100 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f4ffe7067c4d3ebee52b2c69f3c868, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733174239574 2024-12-02T21:17:23,100 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 9137fbe726e141b5ba84f2ffaffc271c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733174239609 2024-12-02T21:17:23,100 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d465b3f1ad34c2f8de254cf2e05c427, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174240765 2024-12-02T21:17:23,104 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting c06c79ace1e947188602e00c4edbd3bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733174239609 2024-12-02T21:17:23,105 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8f9809d1cee48b5bfb1c68e43fe4c57, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174240765 2024-12-02T21:17:23,111 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:23,112 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/8bbba11abe5849c09aac87d103dbc7e0 is 50, key is test_row_0/B:col10/1733174241799/Put/seqid=0 2024-12-02T21:17:23,118 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#A#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:23,118 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/5ee3fd4a68b44521be4ae15feaa8524c is 50, key is test_row_0/A:col10/1733174241799/Put/seqid=0 2024-12-02T21:17:23,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742252_1428 (size=12104) 2024-12-02T21:17:23,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742253_1429 (size=12104) 2024-12-02T21:17:23,533 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/8bbba11abe5849c09aac87d103dbc7e0 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8bbba11abe5849c09aac87d103dbc7e0 2024-12-02T21:17:23,537 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/B of 62c2292167ae6a525d2d66119fcc23eb into 8bbba11abe5849c09aac87d103dbc7e0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:23,537 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:23,537 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/B, priority=13, startTime=1733174243096; duration=0sec 2024-12-02T21:17:23,537 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:23,537 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:B 2024-12-02T21:17:23,537 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:23,538 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:23,538 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/C is initiating minor compaction (all files) 2024-12-02T21:17:23,538 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/C in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:23,538 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/37773f69f41a47d0a545fb1d49c388ff, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c5ad07b87d4747899e593cb1e399b1e4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/6352fa1292b64d598544d97d97b99aa0] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=35.2 K 2024-12-02T21:17:23,539 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 37773f69f41a47d0a545fb1d49c388ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733174239577 2024-12-02T21:17:23,539 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ad07b87d4747899e593cb1e399b1e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733174239609 2024-12-02T21:17:23,540 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6352fa1292b64d598544d97d97b99aa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174240765 2024-12-02T21:17:23,548 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/5ee3fd4a68b44521be4ae15feaa8524c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5ee3fd4a68b44521be4ae15feaa8524c 2024-12-02T21:17:23,554 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/A of 62c2292167ae6a525d2d66119fcc23eb into 5ee3fd4a68b44521be4ae15feaa8524c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:23,554 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:23,554 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/A, priority=13, startTime=1733174243095; duration=0sec 2024-12-02T21:17:23,554 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:23,554 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:A 2024-12-02T21:17:23,555 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#C#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:23,556 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/012b9b2130cb402198a4c9c55637a7c5 is 50, key is test_row_0/C:col10/1733174241799/Put/seqid=0 2024-12-02T21:17:23,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742254_1430 (size=12104) 2024-12-02T21:17:23,576 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/012b9b2130cb402198a4c9c55637a7c5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/012b9b2130cb402198a4c9c55637a7c5 2024-12-02T21:17:23,581 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/C of 62c2292167ae6a525d2d66119fcc23eb into 012b9b2130cb402198a4c9c55637a7c5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:23,581 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:23,581 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/C, priority=13, startTime=1733174243096; duration=0sec 2024-12-02T21:17:23,581 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:23,581 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:C 2024-12-02T21:17:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T21:17:23,729 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:23,730 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-02T21:17:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T21:17:23,731 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:23,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:23,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T21:17:23,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T21:17:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:23,884 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T21:17:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/3f2bb732878649b394fc5d79456e078c is 50, key is test_row_0/A:col10/1733174241907/Put/seqid=0 2024-12-02T21:17:23,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742255_1431 (size=12001) 2024-12-02T21:17:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T21:17:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:24,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174304124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:24,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174304228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:24,291 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/3f2bb732878649b394fc5d79456e078c 2024-12-02T21:17:24,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/aa151c8e70d44c8eb45269be1dc4c073 is 50, key is test_row_0/B:col10/1733174241907/Put/seqid=0 2024-12-02T21:17:24,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742256_1432 (size=12001) 2024-12-02T21:17:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T21:17:24,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174304438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:24,701 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/aa151c8e70d44c8eb45269be1dc4c073 2024-12-02T21:17:24,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/a39d8dad713b4ae5acf60a37c864c992 is 50, key is test_row_0/C:col10/1733174241907/Put/seqid=0 2024-12-02T21:17:24,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742257_1433 (size=12001) 2024-12-02T21:17:24,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174304748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T21:17:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174304958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:24,964 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:24,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174304969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:24,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174304969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:24,971 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:24,972 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:24,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174304979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:24,982 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:25,113 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/a39d8dad713b4ae5acf60a37c864c992 2024-12-02T21:17:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/3f2bb732878649b394fc5d79456e078c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/3f2bb732878649b394fc5d79456e078c 2024-12-02T21:17:25,121 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/3f2bb732878649b394fc5d79456e078c, entries=150, sequenceid=68, filesize=11.7 K 2024-12-02T21:17:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/aa151c8e70d44c8eb45269be1dc4c073 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/aa151c8e70d44c8eb45269be1dc4c073 2024-12-02T21:17:25,133 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/aa151c8e70d44c8eb45269be1dc4c073, entries=150, sequenceid=68, filesize=11.7 K 2024-12-02T21:17:25,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/a39d8dad713b4ae5acf60a37c864c992 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a39d8dad713b4ae5acf60a37c864c992 2024-12-02T21:17:25,141 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a39d8dad713b4ae5acf60a37c864c992, entries=150, sequenceid=68, filesize=11.7 K 2024-12-02T21:17:25,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-02T21:17:25,142 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 62c2292167ae6a525d2d66119fcc23eb in 1258ms, sequenceid=68, compaction requested=false 2024-12-02T21:17:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-02T21:17:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-02T21:17:25,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-02T21:17:25,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-12-02T21:17:25,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4180 sec 2024-12-02T21:17:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:25,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:17:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:25,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9ff1251bf4b54cc6bb9a61f83bc80942 is 50, key is test_row_0/A:col10/1733174244123/Put/seqid=0 2024-12-02T21:17:25,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742258_1434 (size=14341) 2024-12-02T21:17:25,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174305379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174305489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:25,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9ff1251bf4b54cc6bb9a61f83bc80942 2024-12-02T21:17:25,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/0416ad719d6a4cd8ad66c52b8d321638 is 50, key is test_row_0/B:col10/1733174244123/Put/seqid=0 2024-12-02T21:17:25,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742259_1435 (size=12001) 2024-12-02T21:17:25,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/0416ad719d6a4cd8ad66c52b8d321638 2024-12-02T21:17:25,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/c177b7b216204e1bb62e993e333c25d7 is 50, key is test_row_0/C:col10/1733174244123/Put/seqid=0 2024-12-02T21:17:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174305699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:25,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742260_1436 (size=12001) 2024-12-02T21:17:25,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/c177b7b216204e1bb62e993e333c25d7 2024-12-02T21:17:25,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9ff1251bf4b54cc6bb9a61f83bc80942 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9ff1251bf4b54cc6bb9a61f83bc80942 2024-12-02T21:17:25,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9ff1251bf4b54cc6bb9a61f83bc80942, entries=200, sequenceid=82, filesize=14.0 K 2024-12-02T21:17:25,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/0416ad719d6a4cd8ad66c52b8d321638 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/0416ad719d6a4cd8ad66c52b8d321638 2024-12-02T21:17:25,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/0416ad719d6a4cd8ad66c52b8d321638, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T21:17:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/c177b7b216204e1bb62e993e333c25d7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c177b7b216204e1bb62e993e333c25d7 2024-12-02T21:17:25,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c177b7b216204e1bb62e993e333c25d7, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T21:17:25,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 62c2292167ae6a525d2d66119fcc23eb in 466ms, sequenceid=82, compaction requested=true 2024-12-02T21:17:25,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:25,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:25,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:25,728 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:25,728 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:25,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:25,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:25,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:25,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:25,729 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:25,729 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:25,729 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/B is initiating minor compaction (all files) 2024-12-02T21:17:25,729 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/A is initiating minor compaction (all files) 2024-12-02T21:17:25,729 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/B in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:25,729 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/A in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:25,729 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8bbba11abe5849c09aac87d103dbc7e0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/aa151c8e70d44c8eb45269be1dc4c073, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/0416ad719d6a4cd8ad66c52b8d321638] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=35.3 K 2024-12-02T21:17:25,729 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5ee3fd4a68b44521be4ae15feaa8524c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/3f2bb732878649b394fc5d79456e078c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9ff1251bf4b54cc6bb9a61f83bc80942] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=37.5 K 2024-12-02T21:17:25,729 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bbba11abe5849c09aac87d103dbc7e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174240765 2024-12-02T21:17:25,729 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ee3fd4a68b44521be4ae15feaa8524c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174240765 2024-12-02T21:17:25,730 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting aa151c8e70d44c8eb45269be1dc4c073, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733174241898 2024-12-02T21:17:25,730 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f2bb732878649b394fc5d79456e078c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733174241898 2024-12-02T21:17:25,730 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 0416ad719d6a4cd8ad66c52b8d321638, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733174244115 2024-12-02T21:17:25,730 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ff1251bf4b54cc6bb9a61f83bc80942, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733174244110 2024-12-02T21:17:25,736 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#A#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:25,737 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/0dd88167d8d4411b836faf6e99872958 is 50, key is test_row_0/A:col10/1733174244123/Put/seqid=0 2024-12-02T21:17:25,739 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:25,739 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/1970d9b11917472499ebd2c8cfd02a49 is 50, key is test_row_0/B:col10/1733174244123/Put/seqid=0 2024-12-02T21:17:25,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742261_1437 (size=12207) 2024-12-02T21:17:25,752 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/0dd88167d8d4411b836faf6e99872958 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0dd88167d8d4411b836faf6e99872958 2024-12-02T21:17:25,756 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/A of 62c2292167ae6a525d2d66119fcc23eb into 0dd88167d8d4411b836faf6e99872958(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:25,756 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:25,756 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/A, priority=13, startTime=1733174245728; duration=0sec 2024-12-02T21:17:25,756 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:25,756 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:A 2024-12-02T21:17:25,756 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:25,757 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:25,757 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/C is initiating minor compaction (all files) 2024-12-02T21:17:25,757 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/C in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:25,757 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/012b9b2130cb402198a4c9c55637a7c5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a39d8dad713b4ae5acf60a37c864c992, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c177b7b216204e1bb62e993e333c25d7] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=35.3 K 2024-12-02T21:17:25,758 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 012b9b2130cb402198a4c9c55637a7c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733174240765 2024-12-02T21:17:25,758 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting a39d8dad713b4ae5acf60a37c864c992, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733174241898 2024-12-02T21:17:25,758 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting c177b7b216204e1bb62e993e333c25d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733174244115 2024-12-02T21:17:25,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742262_1438 (size=12207) 2024-12-02T21:17:25,774 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:25,774 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/8579e69ea6e2421ebb98a455f23ebc96 is 50, key is test_row_0/C:col10/1733174244123/Put/seqid=0 2024-12-02T21:17:25,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742263_1439 (size=12207) 2024-12-02T21:17:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T21:17:25,879 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:25,879 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-02T21:17:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T21:17:25,881 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:25,882 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:25,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T21:17:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:26,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T21:17:26,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:26,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:26,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:26,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/70dc81f5bb40467281e844d36651f7ed is 50, key is test_row_0/A:col10/1733174245301/Put/seqid=0 2024-12-02T21:17:26,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742264_1440 (size=14341) 2024-12-02T21:17:26,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T21:17:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174306078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:26,173 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/1970d9b11917472499ebd2c8cfd02a49 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/1970d9b11917472499ebd2c8cfd02a49 2024-12-02T21:17:26,177 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/B of 62c2292167ae6a525d2d66119fcc23eb into 1970d9b11917472499ebd2c8cfd02a49(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:26,177 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:26,177 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/B, priority=13, startTime=1733174245728; duration=0sec 2024-12-02T21:17:26,177 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:26,177 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:B 2024-12-02T21:17:26,185 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/8579e69ea6e2421ebb98a455f23ebc96 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/8579e69ea6e2421ebb98a455f23ebc96 2024-12-02T21:17:26,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T21:17:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,189 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/C of 62c2292167ae6a525d2d66119fcc23eb into 8579e69ea6e2421ebb98a455f23ebc96(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:26,189 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,189 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/C, priority=13, startTime=1733174245728; duration=0sec 2024-12-02T21:17:26,189 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:26,189 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:C 2024-12-02T21:17:26,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174306189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T21:17:26,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T21:17:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174306401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:26,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/70dc81f5bb40467281e844d36651f7ed 2024-12-02T21:17:26,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/f8b4c6e42aeb4a60a567ea99b631b7d9 is 50, key is test_row_0/B:col10/1733174245301/Put/seqid=0 2024-12-02T21:17:26,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742265_1441 (size=12001) 2024-12-02T21:17:26,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T21:17:26,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:26,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T21:17:26,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T21:17:26,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:26,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174306709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:26,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T21:17:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:26,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/f8b4c6e42aeb4a60a567ea99b631b7d9 2024-12-02T21:17:26,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/5301089925d9403d8bfb48c5839c866f is 50, key is test_row_0/C:col10/1733174245301/Put/seqid=0 2024-12-02T21:17:26,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742266_1442 (size=12001) 2024-12-02T21:17:26,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/5301089925d9403d8bfb48c5839c866f 2024-12-02T21:17:26,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/70dc81f5bb40467281e844d36651f7ed as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/70dc81f5bb40467281e844d36651f7ed 2024-12-02T21:17:26,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/70dc81f5bb40467281e844d36651f7ed, entries=200, sequenceid=102, filesize=14.0 K 2024-12-02T21:17:26,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/f8b4c6e42aeb4a60a567ea99b631b7d9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f8b4c6e42aeb4a60a567ea99b631b7d9 2024-12-02T21:17:26,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f8b4c6e42aeb4a60a567ea99b631b7d9, entries=150, sequenceid=102, filesize=11.7 K 2024-12-02T21:17:26,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/5301089925d9403d8bfb48c5839c866f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5301089925d9403d8bfb48c5839c866f 2024-12-02T21:17:26,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5301089925d9403d8bfb48c5839c866f, entries=150, sequenceid=102, filesize=11.7 K 2024-12-02T21:17:26,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 62c2292167ae6a525d2d66119fcc23eb in 879ms, sequenceid=102, compaction requested=false 2024-12-02T21:17:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:26,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T21:17:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:26,951 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T21:17:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:26,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/52ae3cd70d824948867c81cf07ecf79d is 50, key is test_row_0/A:col10/1733174246069/Put/seqid=0 2024-12-02T21:17:26,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742267_1443 (size=12001) 2024-12-02T21:17:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T21:17:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:27,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174307339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:27,372 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/52ae3cd70d824948867c81cf07ecf79d 2024-12-02T21:17:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/d6cd962f23d74382a550e92b6fbd0a23 is 50, key is test_row_0/B:col10/1733174246069/Put/seqid=0 2024-12-02T21:17:27,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742268_1444 (size=12001) 2024-12-02T21:17:27,393 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/d6cd962f23d74382a550e92b6fbd0a23 2024-12-02T21:17:27,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/3e76e082be614e23b0cdca3295763238 is 50, key is test_row_0/C:col10/1733174246069/Put/seqid=0 2024-12-02T21:17:27,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742269_1445 (size=12001) 2024-12-02T21:17:27,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174307449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:27,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174307658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:27,805 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/3e76e082be614e23b0cdca3295763238 2024-12-02T21:17:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/52ae3cd70d824948867c81cf07ecf79d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/52ae3cd70d824948867c81cf07ecf79d 2024-12-02T21:17:27,812 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/52ae3cd70d824948867c81cf07ecf79d, entries=150, sequenceid=117, filesize=11.7 K 2024-12-02T21:17:27,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/d6cd962f23d74382a550e92b6fbd0a23 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/d6cd962f23d74382a550e92b6fbd0a23 2024-12-02T21:17:27,815 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/d6cd962f23d74382a550e92b6fbd0a23, entries=150, sequenceid=117, filesize=11.7 K 2024-12-02T21:17:27,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/3e76e082be614e23b0cdca3295763238 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/3e76e082be614e23b0cdca3295763238 2024-12-02T21:17:27,818 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/3e76e082be614e23b0cdca3295763238, entries=150, sequenceid=117, filesize=11.7 K 2024-12-02T21:17:27,819 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 62c2292167ae6a525d2d66119fcc23eb in 867ms, sequenceid=117, compaction requested=true 2024-12-02T21:17:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-02T21:17:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-02T21:17:27,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-02T21:17:27,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9380 sec 2024-12-02T21:17:27,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9410 sec 2024-12-02T21:17:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:27,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T21:17:27,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:27,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:27,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:27,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:27,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:27,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:27,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/af4e76bd53044e9a9a1dac45b3a3fdeb is 50, key is test_row_0/A:col10/1733174247284/Put/seqid=0 2024-12-02T21:17:27,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742270_1446 (size=14541) 2024-12-02T21:17:27,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/af4e76bd53044e9a9a1dac45b3a3fdeb 2024-12-02T21:17:27,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/cc42d58f7e6e41a79def276135548bee is 50, key is test_row_0/B:col10/1733174247284/Put/seqid=0 2024-12-02T21:17:28,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742271_1447 (size=12151) 2024-12-02T21:17:28,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/cc42d58f7e6e41a79def276135548bee 2024-12-02T21:17:28,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/14ed95052d4247919634a4721e6b2654 is 50, key is test_row_0/C:col10/1733174247284/Put/seqid=0 2024-12-02T21:17:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T21:17:28,040 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:28,041 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:28,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742272_1448 (size=12151) 2024-12-02T21:17:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-02T21:17:28,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/14ed95052d4247919634a4721e6b2654 2024-12-02T21:17:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T21:17:28,043 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:28,043 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:28,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:28,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174308041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:28,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/af4e76bd53044e9a9a1dac45b3a3fdeb as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/af4e76bd53044e9a9a1dac45b3a3fdeb 2024-12-02T21:17:28,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/af4e76bd53044e9a9a1dac45b3a3fdeb, entries=200, sequenceid=137, filesize=14.2 K 2024-12-02T21:17:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/cc42d58f7e6e41a79def276135548bee as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/cc42d58f7e6e41a79def276135548bee 2024-12-02T21:17:28,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/cc42d58f7e6e41a79def276135548bee, entries=150, sequenceid=137, filesize=11.9 K 2024-12-02T21:17:28,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/14ed95052d4247919634a4721e6b2654 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/14ed95052d4247919634a4721e6b2654 2024-12-02T21:17:28,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/14ed95052d4247919634a4721e6b2654, entries=150, sequenceid=137, filesize=11.9 K 2024-12-02T21:17:28,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 62c2292167ae6a525d2d66119fcc23eb in 101ms, sequenceid=137, compaction requested=true 2024-12-02T21:17:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:28,075 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:17:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T21:17:28,075 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:17:28,076 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:17:28,076 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:17:28,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/A is initiating minor compaction (all files) 2024-12-02T21:17:28,077 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/B is initiating minor compaction (all files) 2024-12-02T21:17:28,077 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/B in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,077 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/A in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,077 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/1970d9b11917472499ebd2c8cfd02a49, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f8b4c6e42aeb4a60a567ea99b631b7d9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/d6cd962f23d74382a550e92b6fbd0a23, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/cc42d58f7e6e41a79def276135548bee] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=47.2 K 2024-12-02T21:17:28,077 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0dd88167d8d4411b836faf6e99872958, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/70dc81f5bb40467281e844d36651f7ed, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/52ae3cd70d824948867c81cf07ecf79d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/af4e76bd53044e9a9a1dac45b3a3fdeb] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=51.8 K 2024-12-02T21:17:28,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd88167d8d4411b836faf6e99872958, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733174244115 2024-12-02T21:17:28,077 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 1970d9b11917472499ebd2c8cfd02a49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733174244115 2024-12-02T21:17:28,077 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b4c6e42aeb4a60a567ea99b631b7d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733174245301 2024-12-02T21:17:28,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70dc81f5bb40467281e844d36651f7ed, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733174245301 2024-12-02T21:17:28,077 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting d6cd962f23d74382a550e92b6fbd0a23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733174246054 2024-12-02T21:17:28,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ae3cd70d824948867c81cf07ecf79d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733174246054 2024-12-02T21:17:28,078 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting cc42d58f7e6e41a79def276135548bee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733174247284 2024-12-02T21:17:28,078 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting af4e76bd53044e9a9a1dac45b3a3fdeb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733174247284 2024-12-02T21:17:28,090 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#A#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:28,091 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/d8c710425ff84f7cbc328838dd6a3b40 is 50, key is test_row_0/A:col10/1733174247284/Put/seqid=0 2024-12-02T21:17:28,113 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:28,113 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/bb29de8ea51f44609e69b3bb35d416ee is 50, key is test_row_0/B:col10/1733174247284/Put/seqid=0 2024-12-02T21:17:28,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742273_1449 (size=12493) 2024-12-02T21:17:28,130 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/d8c710425ff84f7cbc328838dd6a3b40 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/d8c710425ff84f7cbc328838dd6a3b40 2024-12-02T21:17:28,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742274_1450 (size=12493) 2024-12-02T21:17:28,138 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/A of 62c2292167ae6a525d2d66119fcc23eb into d8c710425ff84f7cbc328838dd6a3b40(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:28,138 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:28,138 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/A, priority=12, startTime=1733174248075; duration=0sec 2024-12-02T21:17:28,138 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:28,138 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:A 2024-12-02T21:17:28,138 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:17:28,139 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/bb29de8ea51f44609e69b3bb35d416ee as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/bb29de8ea51f44609e69b3bb35d416ee 2024-12-02T21:17:28,143 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:17:28,143 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/C is initiating minor compaction (all files) 2024-12-02T21:17:28,143 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/C in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,143 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/8579e69ea6e2421ebb98a455f23ebc96, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5301089925d9403d8bfb48c5839c866f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/3e76e082be614e23b0cdca3295763238, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/14ed95052d4247919634a4721e6b2654] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=47.2 K 2024-12-02T21:17:28,143 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8579e69ea6e2421ebb98a455f23ebc96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733174244115 2024-12-02T21:17:28,144 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5301089925d9403d8bfb48c5839c866f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733174245301 2024-12-02T21:17:28,144 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e76e082be614e23b0cdca3295763238, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733174246054 2024-12-02T21:17:28,144 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14ed95052d4247919634a4721e6b2654, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733174247284 2024-12-02T21:17:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T21:17:28,152 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/B of 62c2292167ae6a525d2d66119fcc23eb into bb29de8ea51f44609e69b3bb35d416ee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:28,152 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:28,152 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/B, priority=12, startTime=1733174248075; duration=0sec 2024-12-02T21:17:28,152 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:28,152 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:B 2024-12-02T21:17:28,162 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#C#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:28,163 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/ec396c292b824da4808b18ec2c93fc6f is 50, key is test_row_0/C:col10/1733174247284/Put/seqid=0 2024-12-02T21:17:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:28,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T21:17:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:28,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/fa8c34f421764f7ca37b58f230a16422 is 50, key is test_row_0/A:col10/1733174248165/Put/seqid=0 2024-12-02T21:17:28,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T21:17:28,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742275_1451 (size=12493) 2024-12-02T21:17:28,215 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/ec396c292b824da4808b18ec2c93fc6f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ec396c292b824da4808b18ec2c93fc6f 2024-12-02T21:17:28,222 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/C of 62c2292167ae6a525d2d66119fcc23eb into ec396c292b824da4808b18ec2c93fc6f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:28,222 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:28,222 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/C, priority=12, startTime=1733174248075; duration=0sec 2024-12-02T21:17:28,222 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:28,222 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:C 2024-12-02T21:17:28,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742276_1452 (size=14541) 2024-12-02T21:17:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174308307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:28,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T21:17:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T21:17:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174308420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:28,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T21:17:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/fa8c34f421764f7ca37b58f230a16422 2024-12-02T21:17:28,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174308640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:28,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/99f42cdb23e747839e908157d46d3c3d is 50, key is test_row_0/B:col10/1733174248165/Put/seqid=0 2024-12-02T21:17:28,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T21:17:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T21:17:28,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742277_1453 (size=12151) 2024-12-02T21:17:28,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/99f42cdb23e747839e908157d46d3c3d 2024-12-02T21:17:28,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/334ffd6552c449fdafd7d351b4beba6c is 50, key is test_row_0/C:col10/1733174248165/Put/seqid=0 2024-12-02T21:17:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742278_1454 (size=12151) 2024-12-02T21:17:28,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T21:17:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174308949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:28,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T21:17:28,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:28,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:28,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:28,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174308978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:28,983 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174308989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:28,994 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:28,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174308989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:28,996 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:29,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174309029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:29,032 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:29,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:29,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:29,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/334ffd6552c449fdafd7d351b4beba6c 2024-12-02T21:17:29,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/fa8c34f421764f7ca37b58f230a16422 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/fa8c34f421764f7ca37b58f230a16422 2024-12-02T21:17:29,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/fa8c34f421764f7ca37b58f230a16422, entries=200, sequenceid=152, filesize=14.2 K 2024-12-02T21:17:29,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/99f42cdb23e747839e908157d46d3c3d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/99f42cdb23e747839e908157d46d3c3d 2024-12-02T21:17:29,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/99f42cdb23e747839e908157d46d3c3d, entries=150, sequenceid=152, filesize=11.9 K 2024-12-02T21:17:29,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/334ffd6552c449fdafd7d351b4beba6c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/334ffd6552c449fdafd7d351b4beba6c 2024-12-02T21:17:29,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/334ffd6552c449fdafd7d351b4beba6c, entries=150, sequenceid=152, filesize=11.9 K 2024-12-02T21:17:29,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 62c2292167ae6a525d2d66119fcc23eb in 942ms, sequenceid=152, compaction requested=false 2024-12-02T21:17:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:29,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T21:17:29,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:29,114 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T21:17:29,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:29,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:29,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:29,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:29,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:29,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:29,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/5bb19d4a47fa4f3396c6e67cedcb81ef is 50, key is test_row_0/A:col10/1733174248305/Put/seqid=0 2024-12-02T21:17:29,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742279_1455 (size=12151) 2024-12-02T21:17:29,124 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/5bb19d4a47fa4f3396c6e67cedcb81ef 2024-12-02T21:17:29,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/f608d0b8c77a4c40b1dff2e3b51ce75d is 50, key is test_row_0/B:col10/1733174248305/Put/seqid=0 2024-12-02T21:17:29,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742280_1456 (size=12151) 2024-12-02T21:17:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T21:17:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:29,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174309526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:29,555 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/f608d0b8c77a4c40b1dff2e3b51ce75d 2024-12-02T21:17:29,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/04595ab0081e4644930db55c3e0045b4 is 50, key is test_row_0/C:col10/1733174248305/Put/seqid=0 2024-12-02T21:17:29,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742281_1457 (size=12151) 2024-12-02T21:17:29,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174309629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:29,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174309838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:29,964 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/04595ab0081e4644930db55c3e0045b4 2024-12-02T21:17:29,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/5bb19d4a47fa4f3396c6e67cedcb81ef as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5bb19d4a47fa4f3396c6e67cedcb81ef 2024-12-02T21:17:29,970 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5bb19d4a47fa4f3396c6e67cedcb81ef, entries=150, sequenceid=172, filesize=11.9 K 2024-12-02T21:17:29,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/f608d0b8c77a4c40b1dff2e3b51ce75d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f608d0b8c77a4c40b1dff2e3b51ce75d 2024-12-02T21:17:29,973 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f608d0b8c77a4c40b1dff2e3b51ce75d, entries=150, sequenceid=172, filesize=11.9 K 2024-12-02T21:17:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/04595ab0081e4644930db55c3e0045b4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04595ab0081e4644930db55c3e0045b4 2024-12-02T21:17:29,976 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04595ab0081e4644930db55c3e0045b4, entries=150, sequenceid=172, filesize=11.9 K 2024-12-02T21:17:29,977 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 62c2292167ae6a525d2d66119fcc23eb in 863ms, sequenceid=172, compaction requested=true 2024-12-02T21:17:29,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:29,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:29,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-02T21:17:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-02T21:17:29,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-02T21:17:29,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9350 sec 2024-12-02T21:17:29,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.9390 sec 2024-12-02T21:17:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:30,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:17:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:30,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:30,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:30,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:30,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/eb30dfa852fe47ac87e92bdcc297078a is 50, key is test_row_0/A:col10/1733174250164/Put/seqid=0 2024-12-02T21:17:30,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742282_1458 (size=14541) 2024-12-02T21:17:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T21:17:30,189 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:30,189 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-02T21:17:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T21:17:30,191 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:30,193 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:30,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T21:17:30,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174310307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:30,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T21:17:30,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:30,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:30,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:30,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:30,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:30,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174310428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:30,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T21:17:30,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:30,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:30,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:30,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:30,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T21:17:30,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/eb30dfa852fe47ac87e92bdcc297078a 2024-12-02T21:17:30,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/4ee96c0f47e54e5bb9834c5a405dfc66 is 50, key is test_row_0/B:col10/1733174250164/Put/seqid=0 2024-12-02T21:17:30,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742283_1459 (size=12151) 2024-12-02T21:17:30,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/4ee96c0f47e54e5bb9834c5a405dfc66 2024-12-02T21:17:30,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/a2170704b02d4f0894a311ee497565d9 is 50, key is test_row_0/C:col10/1733174250164/Put/seqid=0 2024-12-02T21:17:30,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742284_1460 (size=12151) 2024-12-02T21:17:30,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/a2170704b02d4f0894a311ee497565d9 2024-12-02T21:17:30,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/eb30dfa852fe47ac87e92bdcc297078a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/eb30dfa852fe47ac87e92bdcc297078a 2024-12-02T21:17:30,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/eb30dfa852fe47ac87e92bdcc297078a, entries=200, sequenceid=186, filesize=14.2 K 2024-12-02T21:17:30,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/4ee96c0f47e54e5bb9834c5a405dfc66 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4ee96c0f47e54e5bb9834c5a405dfc66 2024-12-02T21:17:30,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4ee96c0f47e54e5bb9834c5a405dfc66, entries=150, sequenceid=186, filesize=11.9 K 2024-12-02T21:17:30,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/a2170704b02d4f0894a311ee497565d9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a2170704b02d4f0894a311ee497565d9 2024-12-02T21:17:30,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a2170704b02d4f0894a311ee497565d9, entries=150, sequenceid=186, filesize=11.9 K 2024-12-02T21:17:30,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 62c2292167ae6a525d2d66119fcc23eb in 461ms, sequenceid=186, compaction requested=true 2024-12-02T21:17:30,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:30,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:30,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:30,626 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:17:30,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:30,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:30,626 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:17:30,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:30,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:30,627 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:17:30,627 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:17:30,627 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/A is initiating minor compaction (all files) 2024-12-02T21:17:30,627 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/B is initiating minor compaction (all files) 2024-12-02T21:17:30,627 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/A in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:30,627 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/B in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:30,627 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/d8c710425ff84f7cbc328838dd6a3b40, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/fa8c34f421764f7ca37b58f230a16422, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5bb19d4a47fa4f3396c6e67cedcb81ef, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/eb30dfa852fe47ac87e92bdcc297078a] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=52.5 K 2024-12-02T21:17:30,627 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/bb29de8ea51f44609e69b3bb35d416ee, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/99f42cdb23e747839e908157d46d3c3d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f608d0b8c77a4c40b1dff2e3b51ce75d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4ee96c0f47e54e5bb9834c5a405dfc66] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=47.8 K 2024-12-02T21:17:30,628 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8c710425ff84f7cbc328838dd6a3b40, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733174247284 2024-12-02T21:17:30,628 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting bb29de8ea51f44609e69b3bb35d416ee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733174247284 2024-12-02T21:17:30,628 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa8c34f421764f7ca37b58f230a16422, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733174248011 2024-12-02T21:17:30,628 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 99f42cdb23e747839e908157d46d3c3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733174248026 2024-12-02T21:17:30,628 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bb19d4a47fa4f3396c6e67cedcb81ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733174248177 2024-12-02T21:17:30,628 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f608d0b8c77a4c40b1dff2e3b51ce75d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733174248177 2024-12-02T21:17:30,628 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb30dfa852fe47ac87e92bdcc297078a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733174249488 2024-12-02T21:17:30,629 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ee96c0f47e54e5bb9834c5a405dfc66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733174249517 2024-12-02T21:17:30,637 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#A#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:30,638 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/0b58aad345b94e9590d13a554fbe5422 is 50, key is test_row_0/A:col10/1733174250164/Put/seqid=0 2024-12-02T21:17:30,641 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:30,642 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/a31670ef5ffc4c93a79b56ff9a0abc7e is 50, key is test_row_0/B:col10/1733174250164/Put/seqid=0 2024-12-02T21:17:30,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742285_1461 (size=12629) 2024-12-02T21:17:30,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T21:17:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:30,650 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/0b58aad345b94e9590d13a554fbe5422 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0b58aad345b94e9590d13a554fbe5422 2024-12-02T21:17:30,650 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T21:17:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:30,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:30,655 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/A of 62c2292167ae6a525d2d66119fcc23eb into 0b58aad345b94e9590d13a554fbe5422(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:30,655 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:30,655 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/A, priority=12, startTime=1733174250626; duration=0sec 2024-12-02T21:17:30,655 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:30,655 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:A 2024-12-02T21:17:30,655 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T21:17:30,657 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T21:17:30,657 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/C is initiating minor compaction (all files) 2024-12-02T21:17:30,657 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/C in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:30,657 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ec396c292b824da4808b18ec2c93fc6f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/334ffd6552c449fdafd7d351b4beba6c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04595ab0081e4644930db55c3e0045b4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a2170704b02d4f0894a311ee497565d9] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=47.8 K 2024-12-02T21:17:30,657 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec396c292b824da4808b18ec2c93fc6f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733174247284 2024-12-02T21:17:30,657 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 334ffd6552c449fdafd7d351b4beba6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733174248026 2024-12-02T21:17:30,657 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04595ab0081e4644930db55c3e0045b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733174248177 2024-12-02T21:17:30,658 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2170704b02d4f0894a311ee497565d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733174249517 2024-12-02T21:17:30,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9f5c4264afbc4e0993bc4cfa40131b26 is 50, key is test_row_0/A:col10/1733174250274/Put/seqid=0 2024-12-02T21:17:30,671 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#C#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:30,671 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/7da001c218c044e28227f9771ff9ed4c is 50, key is test_row_0/C:col10/1733174250164/Put/seqid=0 2024-12-02T21:17:30,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742287_1463 (size=12151) 2024-12-02T21:17:30,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742286_1462 (size=12629) 2024-12-02T21:17:30,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742288_1464 (size=12629) 2024-12-02T21:17:30,714 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/a31670ef5ffc4c93a79b56ff9a0abc7e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/a31670ef5ffc4c93a79b56ff9a0abc7e 2024-12-02T21:17:30,717 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/7da001c218c044e28227f9771ff9ed4c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7da001c218c044e28227f9771ff9ed4c 2024-12-02T21:17:30,720 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/B of 62c2292167ae6a525d2d66119fcc23eb into a31670ef5ffc4c93a79b56ff9a0abc7e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:30,720 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:30,720 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/B, priority=12, startTime=1733174250626; duration=0sec 2024-12-02T21:17:30,720 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:30,720 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:B 2024-12-02T21:17:30,724 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/C of 62c2292167ae6a525d2d66119fcc23eb into 7da001c218c044e28227f9771ff9ed4c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:30,724 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:30,724 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/C, priority=12, startTime=1733174250626; duration=0sec 2024-12-02T21:17:30,724 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:30,724 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:C 2024-12-02T21:17:30,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174310748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T21:17:30,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174310859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:31,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174311068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:31,089 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9f5c4264afbc4e0993bc4cfa40131b26 2024-12-02T21:17:31,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/49557926a7ac4e4bbbfbae1036585312 is 50, key is test_row_0/B:col10/1733174250274/Put/seqid=0 2024-12-02T21:17:31,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742289_1465 (size=12151) 2024-12-02T21:17:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T21:17:31,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174311378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:31,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:31,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:31,500 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/49557926a7ac4e4bbbfbae1036585312 2024-12-02T21:17:31,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/f126cb81eaf8445c8bd07160e2cec508 is 50, key is test_row_0/C:col10/1733174250274/Put/seqid=0 2024-12-02T21:17:31,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742290_1466 (size=12151) 2024-12-02T21:17:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174311889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:31,910 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/f126cb81eaf8445c8bd07160e2cec508 2024-12-02T21:17:31,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9f5c4264afbc4e0993bc4cfa40131b26 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9f5c4264afbc4e0993bc4cfa40131b26 2024-12-02T21:17:31,918 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9f5c4264afbc4e0993bc4cfa40131b26, entries=150, sequenceid=204, filesize=11.9 K 2024-12-02T21:17:31,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/49557926a7ac4e4bbbfbae1036585312 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/49557926a7ac4e4bbbfbae1036585312 2024-12-02T21:17:31,923 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/49557926a7ac4e4bbbfbae1036585312, entries=150, sequenceid=204, filesize=11.9 K 2024-12-02T21:17:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/f126cb81eaf8445c8bd07160e2cec508 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/f126cb81eaf8445c8bd07160e2cec508 2024-12-02T21:17:31,936 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/f126cb81eaf8445c8bd07160e2cec508, entries=150, sequenceid=204, filesize=11.9 K 2024-12-02T21:17:31,937 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 62c2292167ae6a525d2d66119fcc23eb in 1287ms, sequenceid=204, compaction requested=false 2024-12-02T21:17:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-02T21:17:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-02T21:17:31,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-02T21:17:31,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7470 sec 2024-12-02T21:17:31,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7570 sec 2024-12-02T21:17:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T21:17:32,339 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:32,339 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:32,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-02T21:17:32,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T21:17:32,341 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:32,341 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:32,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T21:17:32,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T21:17:32,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:32,493 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:17:32,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:32,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/ea32a3fca2594d55afde4bc999483f3e is 50, key is test_row_0/A:col10/1733174250722/Put/seqid=0 2024-12-02T21:17:32,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742291_1467 (size=12151) 2024-12-02T21:17:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T21:17:32,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:32,912 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/ea32a3fca2594d55afde4bc999483f3e 2024-12-02T21:17:32,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/3348dac321e446819b49926640a0ed22 is 50, key is test_row_0/B:col10/1733174250722/Put/seqid=0 2024-12-02T21:17:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742292_1468 (size=12151) 2024-12-02T21:17:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T21:17:33,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174312996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:33,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174313108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:33,323 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/3348dac321e446819b49926640a0ed22 2024-12-02T21:17:33,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174313319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/d61e6903792f4bd38f22f59c6cc27a1d is 50, key is test_row_0/C:col10/1733174250722/Put/seqid=0 2024-12-02T21:17:33,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742293_1469 (size=12151) 2024-12-02T21:17:33,362 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/d61e6903792f4bd38f22f59c6cc27a1d 2024-12-02T21:17:33,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/ea32a3fca2594d55afde4bc999483f3e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/ea32a3fca2594d55afde4bc999483f3e 2024-12-02T21:17:33,374 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/ea32a3fca2594d55afde4bc999483f3e, entries=150, sequenceid=221, filesize=11.9 K 2024-12-02T21:17:33,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/3348dac321e446819b49926640a0ed22 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/3348dac321e446819b49926640a0ed22 2024-12-02T21:17:33,383 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/3348dac321e446819b49926640a0ed22, entries=150, sequenceid=221, filesize=11.9 K 2024-12-02T21:17:33,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/d61e6903792f4bd38f22f59c6cc27a1d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/d61e6903792f4bd38f22f59c6cc27a1d 2024-12-02T21:17:33,395 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/d61e6903792f4bd38f22f59c6cc27a1d, entries=150, sequenceid=221, filesize=11.9 K 2024-12-02T21:17:33,396 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 62c2292167ae6a525d2d66119fcc23eb in 903ms, sequenceid=221, compaction requested=true 2024-12-02T21:17:33,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:33,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:33,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-02T21:17:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-02T21:17:33,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-02T21:17:33,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0580 sec 2024-12-02T21:17:33,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0630 sec 2024-12-02T21:17:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T21:17:33,479 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:33,479 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-02T21:17:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T21:17:33,481 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:33,482 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:33,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T21:17:33,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-02T21:17:33,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:33,635 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T21:17:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:33,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:33,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:33,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:33,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:33,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:33,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/df1e525375f94a46b7e891ff636e2a2e is 50, key is test_row_0/A:col10/1733174252953/Put/seqid=0 2024-12-02T21:17:33,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742294_1470 (size=14541) 2024-12-02T21:17:33,674 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/df1e525375f94a46b7e891ff636e2a2e 2024-12-02T21:17:33,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/4164482f8f64427380bc5847bc17c5a4 is 50, key is test_row_0/B:col10/1733174252953/Put/seqid=0 2024-12-02T21:17:33,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742295_1471 (size=12151) 2024-12-02T21:17:33,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174313742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T21:17:33,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174313858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:34,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174314079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T21:17:34,137 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/4164482f8f64427380bc5847bc17c5a4 2024-12-02T21:17:34,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/cd6a7298989a47edb7725f6413f43342 is 50, key is test_row_0/C:col10/1733174252953/Put/seqid=0 2024-12-02T21:17:34,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742296_1472 (size=12151) 2024-12-02T21:17:34,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:34,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174314399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:34,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:34,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:34,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:34,556 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/cd6a7298989a47edb7725f6413f43342 2024-12-02T21:17:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/df1e525375f94a46b7e891ff636e2a2e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/df1e525375f94a46b7e891ff636e2a2e 2024-12-02T21:17:34,567 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/df1e525375f94a46b7e891ff636e2a2e, entries=200, sequenceid=240, filesize=14.2 K 2024-12-02T21:17:34,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/4164482f8f64427380bc5847bc17c5a4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4164482f8f64427380bc5847bc17c5a4 2024-12-02T21:17:34,573 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4164482f8f64427380bc5847bc17c5a4, entries=150, sequenceid=240, filesize=11.9 K 2024-12-02T21:17:34,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/cd6a7298989a47edb7725f6413f43342 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/cd6a7298989a47edb7725f6413f43342 2024-12-02T21:17:34,578 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/cd6a7298989a47edb7725f6413f43342, entries=150, sequenceid=240, filesize=11.9 K 2024-12-02T21:17:34,578 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 62c2292167ae6a525d2d66119fcc23eb in 944ms, sequenceid=240, compaction requested=true 2024-12-02T21:17:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-02T21:17:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-02T21:17:34,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-02T21:17:34,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1000 sec 2024-12-02T21:17:34,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1060 sec 2024-12-02T21:17:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T21:17:34,619 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:34,620 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-02T21:17:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T21:17:34,622 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:34,623 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:34,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T21:17:34,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T21:17:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:34,775 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T21:17:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/f6a912a6c5354aff91948158594cb03e is 50, key is test_row_0/A:col10/1733174253739/Put/seqid=0 2024-12-02T21:17:34,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742297_1473 (size=12151) 2024-12-02T21:17:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T21:17:35,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174315031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:35,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174315139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:35,194 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/f6a912a6c5354aff91948158594cb03e 2024-12-02T21:17:35,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/570d5f96a9154b7da89eddeea1410e8b is 50, key is test_row_0/B:col10/1733174253739/Put/seqid=0 2024-12-02T21:17:35,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742298_1474 (size=12151) 2024-12-02T21:17:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T21:17:35,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174315349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:35,602 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/570d5f96a9154b7da89eddeea1410e8b 2024-12-02T21:17:35,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/223f9418244643159d442296078fc248 is 50, key is test_row_0/C:col10/1733174253739/Put/seqid=0 2024-12-02T21:17:35,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742299_1475 (size=12151) 2024-12-02T21:17:35,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174315659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T21:17:36,011 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/223f9418244643159d442296078fc248 2024-12-02T21:17:36,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/f6a912a6c5354aff91948158594cb03e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/f6a912a6c5354aff91948158594cb03e 2024-12-02T21:17:36,020 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/f6a912a6c5354aff91948158594cb03e, entries=150, sequenceid=253, filesize=11.9 K 2024-12-02T21:17:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/570d5f96a9154b7da89eddeea1410e8b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/570d5f96a9154b7da89eddeea1410e8b 2024-12-02T21:17:36,024 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/570d5f96a9154b7da89eddeea1410e8b, entries=150, sequenceid=253, filesize=11.9 K 2024-12-02T21:17:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/223f9418244643159d442296078fc248 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/223f9418244643159d442296078fc248 2024-12-02T21:17:36,028 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/223f9418244643159d442296078fc248, entries=150, sequenceid=253, filesize=11.9 K 2024-12-02T21:17:36,029 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 62c2292167ae6a525d2d66119fcc23eb in 1254ms, sequenceid=253, compaction requested=true 2024-12-02T21:17:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-02T21:17:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-02T21:17:36,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-02T21:17:36,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-12-02T21:17:36,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4160 sec 2024-12-02T21:17:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:36,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T21:17:36,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:36,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:36,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:36,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:36,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:36,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:36,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/076e6ff7af2741edb2a1d4468581706c is 50, key is test_row_0/A:col10/1733174256171/Put/seqid=0 2024-12-02T21:17:36,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742300_1476 (size=14741) 2024-12-02T21:17:36,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174316231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:36,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174316349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174316559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:36,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/076e6ff7af2741edb2a1d4468581706c 2024-12-02T21:17:36,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/56bf4344b8694f019edd6420bf9ec023 is 50, key is test_row_0/B:col10/1733174256171/Put/seqid=0 2024-12-02T21:17:36,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742301_1477 (size=12301) 2024-12-02T21:17:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T21:17:36,769 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:36,769 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-02T21:17:36,771 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T21:17:36,772 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:36,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T21:17:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174316879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:36,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T21:17:36,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:36,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:36,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:36,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:36,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:36,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/56bf4344b8694f019edd6420bf9ec023 2024-12-02T21:17:37,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/7f5e4ee402ed441587705d24569a624b is 50, key is test_row_0/C:col10/1733174256171/Put/seqid=0 2024-12-02T21:17:37,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742302_1478 (size=12301) 2024-12-02T21:17:37,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/7f5e4ee402ed441587705d24569a624b 2024-12-02T21:17:37,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/076e6ff7af2741edb2a1d4468581706c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/076e6ff7af2741edb2a1d4468581706c 2024-12-02T21:17:37,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/076e6ff7af2741edb2a1d4468581706c, entries=200, sequenceid=273, filesize=14.4 K 2024-12-02T21:17:37,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/56bf4344b8694f019edd6420bf9ec023 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/56bf4344b8694f019edd6420bf9ec023 2024-12-02T21:17:37,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/56bf4344b8694f019edd6420bf9ec023, entries=150, sequenceid=273, filesize=12.0 K 2024-12-02T21:17:37,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/7f5e4ee402ed441587705d24569a624b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7f5e4ee402ed441587705d24569a624b 2024-12-02T21:17:37,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7f5e4ee402ed441587705d24569a624b, entries=150, sequenceid=273, filesize=12.0 K 2024-12-02T21:17:37,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 62c2292167ae6a525d2d66119fcc23eb in 868ms, sequenceid=273, compaction requested=true 2024-12-02T21:17:37,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:37,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:37,041 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T21:17:37,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:37,041 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T21:17:37,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:37,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:37,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:37,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:37,044 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 78364 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T21:17:37,044 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/A is initiating minor compaction (all files) 2024-12-02T21:17:37,044 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/A in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:37,044 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0b58aad345b94e9590d13a554fbe5422, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9f5c4264afbc4e0993bc4cfa40131b26, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/ea32a3fca2594d55afde4bc999483f3e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/df1e525375f94a46b7e891ff636e2a2e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/f6a912a6c5354aff91948158594cb03e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/076e6ff7af2741edb2a1d4468581706c] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=76.5 K 2024-12-02T21:17:37,044 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b58aad345b94e9590d13a554fbe5422, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733174249517 2024-12-02T21:17:37,044 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73534 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T21:17:37,044 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/B is initiating minor compaction (all files) 2024-12-02T21:17:37,045 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/B in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:37,045 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/a31670ef5ffc4c93a79b56ff9a0abc7e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/49557926a7ac4e4bbbfbae1036585312, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/3348dac321e446819b49926640a0ed22, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4164482f8f64427380bc5847bc17c5a4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/570d5f96a9154b7da89eddeea1410e8b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/56bf4344b8694f019edd6420bf9ec023] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=71.8 K 2024-12-02T21:17:37,045 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f5c4264afbc4e0993bc4cfa40131b26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733174250260 2024-12-02T21:17:37,045 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting a31670ef5ffc4c93a79b56ff9a0abc7e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733174249517 2024-12-02T21:17:37,045 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea32a3fca2594d55afde4bc999483f3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733174250722 2024-12-02T21:17:37,045 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 49557926a7ac4e4bbbfbae1036585312, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733174250260 2024-12-02T21:17:37,046 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting df1e525375f94a46b7e891ff636e2a2e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733174252953 2024-12-02T21:17:37,046 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 3348dac321e446819b49926640a0ed22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733174250722 2024-12-02T21:17:37,046 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6a912a6c5354aff91948158594cb03e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733174253693 2024-12-02T21:17:37,046 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 4164482f8f64427380bc5847bc17c5a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733174252953 2024-12-02T21:17:37,046 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 076e6ff7af2741edb2a1d4468581706c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174254989 2024-12-02T21:17:37,047 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 570d5f96a9154b7da89eddeea1410e8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733174253693 2024-12-02T21:17:37,047 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 56bf4344b8694f019edd6420bf9ec023, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174255017 2024-12-02T21:17:37,061 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#A#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:37,061 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/2b5c092baf3948fa96e0cb1d74bd2a8a is 50, key is test_row_0/A:col10/1733174256171/Put/seqid=0 2024-12-02T21:17:37,064 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:37,064 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/5705dcc8138442e2b8d3b9ddbbdbcdac is 50, key is test_row_0/B:col10/1733174256171/Put/seqid=0 2024-12-02T21:17:37,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742304_1480 (size=12983) 2024-12-02T21:17:37,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742303_1479 (size=12983) 2024-12-02T21:17:37,074 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/2b5c092baf3948fa96e0cb1d74bd2a8a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/2b5c092baf3948fa96e0cb1d74bd2a8a 2024-12-02T21:17:37,077 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/A of 62c2292167ae6a525d2d66119fcc23eb into 2b5c092baf3948fa96e0cb1d74bd2a8a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:37,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:37,077 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/A, priority=10, startTime=1733174257041; duration=0sec 2024-12-02T21:17:37,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:37,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:A 2024-12-02T21:17:37,077 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T21:17:37,078 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73534 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T21:17:37,078 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/C is initiating minor compaction (all files) 2024-12-02T21:17:37,078 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/C in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:37,078 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7da001c218c044e28227f9771ff9ed4c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/f126cb81eaf8445c8bd07160e2cec508, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/d61e6903792f4bd38f22f59c6cc27a1d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/cd6a7298989a47edb7725f6413f43342, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/223f9418244643159d442296078fc248, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7f5e4ee402ed441587705d24569a624b] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=71.8 K 2024-12-02T21:17:37,079 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7da001c218c044e28227f9771ff9ed4c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733174249517 2024-12-02T21:17:37,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T21:17:37,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:37,079 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting f126cb81eaf8445c8bd07160e2cec508, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733174250260 2024-12-02T21:17:37,079 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T21:17:37,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:37,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:37,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:37,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:37,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:37,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:37,080 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting d61e6903792f4bd38f22f59c6cc27a1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733174250722 2024-12-02T21:17:37,080 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd6a7298989a47edb7725f6413f43342, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733174252953 2024-12-02T21:17:37,081 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 223f9418244643159d442296078fc248, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733174253693 2024-12-02T21:17:37,081 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f5e4ee402ed441587705d24569a624b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174255017 2024-12-02T21:17:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/0e2f8e22c3da45fb8237be27d492f364 is 50, key is test_row_0/A:col10/1733174256230/Put/seqid=0 2024-12-02T21:17:37,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742305_1481 (size=12301) 2024-12-02T21:17:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T21:17:37,093 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:37,094 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/ee88a23ac3d34172b594eb868b8169d1 is 50, key is test_row_0/C:col10/1733174256171/Put/seqid=0 2024-12-02T21:17:37,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742306_1482 (size=12983) 2024-12-02T21:17:37,111 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/ee88a23ac3d34172b594eb868b8169d1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ee88a23ac3d34172b594eb868b8169d1 2024-12-02T21:17:37,114 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/C of 62c2292167ae6a525d2d66119fcc23eb into ee88a23ac3d34172b594eb868b8169d1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:37,114 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:37,114 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/C, priority=10, startTime=1733174257042; duration=0sec 2024-12-02T21:17:37,114 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:37,114 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:C 2024-12-02T21:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:37,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T21:17:37,470 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/5705dcc8138442e2b8d3b9ddbbdbcdac as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/5705dcc8138442e2b8d3b9ddbbdbcdac 2024-12-02T21:17:37,474 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/B of 62c2292167ae6a525d2d66119fcc23eb into 5705dcc8138442e2b8d3b9ddbbdbcdac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:37,474 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:37,474 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/B, priority=10, startTime=1733174257041; duration=0sec 2024-12-02T21:17:37,474 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:37,474 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:B 2024-12-02T21:17:37,486 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/0e2f8e22c3da45fb8237be27d492f364 2024-12-02T21:17:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/51a229a8b42a47b2b7d02924baeca768 is 50, key is test_row_0/B:col10/1733174256230/Put/seqid=0 2024-12-02T21:17:37,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742307_1483 (size=12301) 2024-12-02T21:17:37,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174317506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:37,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174317619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:37,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174317828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:37,894 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/51a229a8b42a47b2b7d02924baeca768 2024-12-02T21:17:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/7fdb3b4d50a5452b9b594c3006626883 is 50, key is test_row_0/C:col10/1733174256230/Put/seqid=0 2024-12-02T21:17:37,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742308_1484 (size=12301) 2024-12-02T21:17:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T21:17:38,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174318139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:38,304 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/7fdb3b4d50a5452b9b594c3006626883 2024-12-02T21:17:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/0e2f8e22c3da45fb8237be27d492f364 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0e2f8e22c3da45fb8237be27d492f364 2024-12-02T21:17:38,312 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0e2f8e22c3da45fb8237be27d492f364, entries=150, sequenceid=286, filesize=12.0 K 2024-12-02T21:17:38,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/51a229a8b42a47b2b7d02924baeca768 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/51a229a8b42a47b2b7d02924baeca768 2024-12-02T21:17:38,316 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/51a229a8b42a47b2b7d02924baeca768, entries=150, sequenceid=286, filesize=12.0 K 2024-12-02T21:17:38,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/7fdb3b4d50a5452b9b594c3006626883 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7fdb3b4d50a5452b9b594c3006626883 2024-12-02T21:17:38,320 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7fdb3b4d50a5452b9b594c3006626883, entries=150, sequenceid=286, filesize=12.0 K 2024-12-02T21:17:38,321 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 62c2292167ae6a525d2d66119fcc23eb in 1242ms, sequenceid=286, compaction requested=false 2024-12-02T21:17:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-02T21:17:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-02T21:17:38,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-02T21:17:38,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5500 sec 2024-12-02T21:17:38,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.5550 sec 2024-12-02T21:17:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:38,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T21:17:38,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:38,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:17:38,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:38,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:17:38,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:38,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:17:38,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9c8d7099519549978576608bd39b5c7d is 50, key is test_row_0/A:col10/1733174258651/Put/seqid=0 2024-12-02T21:17:38,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742309_1485 (size=14741) 2024-12-02T21:17:38,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174318685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:38,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174318798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T21:17:38,919 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:38,919 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-02T21:17:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T21:17:38,922 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:38,922 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:38,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:39,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174319009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T21:17:39,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39442 deadline: 1733174319028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:39,038 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:39,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9c8d7099519549978576608bd39b5c7d 2024-12-02T21:17:39,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/8d103300da434b8b8fc41cf4178f0a09 is 50, key is test_row_0/B:col10/1733174258651/Put/seqid=0 2024-12-02T21:17:39,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742310_1486 (size=12301) 2024-12-02T21:17:39,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T21:17:39,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:39,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:39,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:39,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39482 deadline: 1733174319079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:39,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39452 deadline: 1733174319079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:39,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39506 deadline: 1733174319079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:39,086 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18312 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:39,086 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_0' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18310 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:39,086 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 62c2292167ae6a525d2d66119fcc23eb of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18311 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:17:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:39,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T21:17:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T21:17:39,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39492 deadline: 1733174319319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 , the old value is region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., hostname=87c3fdb6c570,35563,1733174113859, seqNum=2 because the exception is null or not the one we care about 2024-12-02T21:17:39,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T21:17:39,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:39,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/8d103300da434b8b8fc41cf4178f0a09 2024-12-02T21:17:39,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/5576cc7a5ac14a5cbf77153c7f489386 is 50, key is test_row_0/C:col10/1733174258651/Put/seqid=0 2024-12-02T21:17:39,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742311_1487 (size=12301) 2024-12-02T21:17:39,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/5576cc7a5ac14a5cbf77153c7f489386 2024-12-02T21:17:39,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9c8d7099519549978576608bd39b5c7d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9c8d7099519549978576608bd39b5c7d 2024-12-02T21:17:39,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9c8d7099519549978576608bd39b5c7d, entries=200, sequenceid=309, filesize=14.4 K 2024-12-02T21:17:39,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/8d103300da434b8b8fc41cf4178f0a09 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8d103300da434b8b8fc41cf4178f0a09 2024-12-02T21:17:39,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8d103300da434b8b8fc41cf4178f0a09, entries=150, sequenceid=309, filesize=12.0 K 2024-12-02T21:17:39,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/5576cc7a5ac14a5cbf77153c7f489386 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5576cc7a5ac14a5cbf77153c7f489386 2024-12-02T21:17:39,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5576cc7a5ac14a5cbf77153c7f489386, entries=150, sequenceid=309, filesize=12.0 K 2024-12-02T21:17:39,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 62c2292167ae6a525d2d66119fcc23eb in 857ms, sequenceid=309, compaction requested=true 2024-12-02T21:17:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:39,509 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62c2292167ae6a525d2d66119fcc23eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T21:17:39,510 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:39,512 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:39,513 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/A is initiating minor compaction (all files) 2024-12-02T21:17:39,513 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/A in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,513 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/2b5c092baf3948fa96e0cb1d74bd2a8a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0e2f8e22c3da45fb8237be27d492f364, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9c8d7099519549978576608bd39b5c7d] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=39.1 K 2024-12-02T21:17:39,513 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:39,513 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/B is initiating minor compaction (all files) 2024-12-02T21:17:39,513 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/B in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,513 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/5705dcc8138442e2b8d3b9ddbbdbcdac, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/51a229a8b42a47b2b7d02924baeca768, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8d103300da434b8b8fc41cf4178f0a09] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=36.7 K 2024-12-02T21:17:39,513 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b5c092baf3948fa96e0cb1d74bd2a8a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174255017 2024-12-02T21:17:39,514 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5705dcc8138442e2b8d3b9ddbbdbcdac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174255017 2024-12-02T21:17:39,514 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e2f8e22c3da45fb8237be27d492f364, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733174256214 2024-12-02T21:17:39,514 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a229a8b42a47b2b7d02924baeca768, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733174256214 2024-12-02T21:17:39,514 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c8d7099519549978576608bd39b5c7d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733174257494 2024-12-02T21:17:39,514 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d103300da434b8b8fc41cf4178f0a09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733174257494 2024-12-02T21:17:39,520 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#A#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:39,520 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/b3ee44901eaa4428abeebdd1566a9cb4 is 50, key is test_row_0/A:col10/1733174258651/Put/seqid=0 2024-12-02T21:17:39,522 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:39,522 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/b62c5dcd8d0443e3bf630ee1b548c274 is 50, key is test_row_0/B:col10/1733174258651/Put/seqid=0 2024-12-02T21:17:39,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T21:17:39,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,531 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:17:39,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:39,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:39,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742312_1488 (size=13085) 2024-12-02T21:17:39,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742313_1489 (size=13085) 2024-12-02T21:17:39,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/31bd6964648243d39f44bbaaeda72c6c is 50, key is test_row_0/A:col10/1733174258663/Put/seqid=0 2024-12-02T21:17:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T21:17:39,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742314_1490 (size=12301) 2024-12-02T21:17:39,571 INFO [Thread-1899 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1899. 2024-12-02T21:17:39,571 DEBUG [Thread-1899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:39,571 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,572 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:39,573 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-12-02T21:17:39,573 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:39,573 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,573 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:39,575 INFO [Thread-1907 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1907. 2024-12-02T21:17:39,575 DEBUG [Thread-1907 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:39,575 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,575 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:39,577 INFO [Thread-1905 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1905. 2024-12-02T21:17:39,577 DEBUG [Thread-1905 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:39,577 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,577 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,577 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:39,578 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-12-02T21:17:39,578 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:39,578 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,578 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:39,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. as already flushing 2024-12-02T21:17:39,840 INFO [Thread-1890 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1890. 2024-12-02T21:17:39,840 DEBUG [Thread-1890 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:39,840 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,840 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:39,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:39,936 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/b3ee44901eaa4428abeebdd1566a9cb4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/b3ee44901eaa4428abeebdd1566a9cb4 2024-12-02T21:17:39,940 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/A of 62c2292167ae6a525d2d66119fcc23eb into b3ee44901eaa4428abeebdd1566a9cb4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:39,940 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:39,940 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/A, priority=13, startTime=1733174259509; duration=0sec 2024-12-02T21:17:39,940 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:39,940 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:A 2024-12-02T21:17:39,940 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:39,941 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:39,941 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 62c2292167ae6a525d2d66119fcc23eb/C is initiating minor compaction (all files) 2024-12-02T21:17:39,941 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62c2292167ae6a525d2d66119fcc23eb/C in TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:39,941 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ee88a23ac3d34172b594eb868b8169d1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7fdb3b4d50a5452b9b594c3006626883, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5576cc7a5ac14a5cbf77153c7f489386] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp, totalSize=36.7 K 2024-12-02T21:17:39,941 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee88a23ac3d34172b594eb868b8169d1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733174255017 2024-12-02T21:17:39,941 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fdb3b4d50a5452b9b594c3006626883, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733174256214 2024-12-02T21:17:39,941 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5576cc7a5ac14a5cbf77153c7f489386, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733174257494 2024-12-02T21:17:39,945 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/b62c5dcd8d0443e3bf630ee1b548c274 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/b62c5dcd8d0443e3bf630ee1b548c274 2024-12-02T21:17:39,946 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62c2292167ae6a525d2d66119fcc23eb#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:39,946 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/56c0e3cd35a1401f926a11421c479a8f is 50, key is test_row_0/C:col10/1733174258651/Put/seqid=0 2024-12-02T21:17:39,948 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/B of 62c2292167ae6a525d2d66119fcc23eb into b62c5dcd8d0443e3bf630ee1b548c274(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:39,948 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:39,948 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/B, priority=13, startTime=1733174259509; duration=0sec 2024-12-02T21:17:39,948 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:39,948 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:B 2024-12-02T21:17:39,951 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/31bd6964648243d39f44bbaaeda72c6c 2024-12-02T21:17:39,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742315_1491 (size=13085) 2024-12-02T21:17:39,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/74afed80d3e14c3ca635b8ae872dbdb0 is 50, key is test_row_0/B:col10/1733174258663/Put/seqid=0 2024-12-02T21:17:39,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742316_1492 (size=12301) 2024-12-02T21:17:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T21:17:40,355 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/56c0e3cd35a1401f926a11421c479a8f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/56c0e3cd35a1401f926a11421c479a8f 2024-12-02T21:17:40,358 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62c2292167ae6a525d2d66119fcc23eb/C of 62c2292167ae6a525d2d66119fcc23eb into 56c0e3cd35a1401f926a11421c479a8f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:40,358 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:40,358 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., storeName=62c2292167ae6a525d2d66119fcc23eb/C, priority=13, startTime=1733174259509; duration=0sec 2024-12-02T21:17:40,358 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:40,358 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62c2292167ae6a525d2d66119fcc23eb:C 2024-12-02T21:17:40,362 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/74afed80d3e14c3ca635b8ae872dbdb0 2024-12-02T21:17:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/04191f517f07416ea7af3f6e80a1cab1 is 50, key is test_row_0/C:col10/1733174258663/Put/seqid=0 2024-12-02T21:17:40,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742317_1493 (size=12301) 2024-12-02T21:17:40,769 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/04191f517f07416ea7af3f6e80a1cab1 2024-12-02T21:17:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/31bd6964648243d39f44bbaaeda72c6c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/31bd6964648243d39f44bbaaeda72c6c 2024-12-02T21:17:40,777 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/31bd6964648243d39f44bbaaeda72c6c, entries=150, sequenceid=320, filesize=12.0 K 2024-12-02T21:17:40,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/74afed80d3e14c3ca635b8ae872dbdb0 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/74afed80d3e14c3ca635b8ae872dbdb0 2024-12-02T21:17:40,780 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/74afed80d3e14c3ca635b8ae872dbdb0, entries=150, sequenceid=320, filesize=12.0 K 2024-12-02T21:17:40,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/04191f517f07416ea7af3f6e80a1cab1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04191f517f07416ea7af3f6e80a1cab1 2024-12-02T21:17:40,783 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04191f517f07416ea7af3f6e80a1cab1, entries=150, sequenceid=320, filesize=12.0 K 2024-12-02T21:17:40,783 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=6.71 KB/6870 for 62c2292167ae6a525d2d66119fcc23eb in 1252ms, sequenceid=320, compaction requested=false 2024-12-02T21:17:40,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 62c2292167ae6a525d2d66119fcc23eb: 2024-12-02T21:17:40,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:40,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-02T21:17:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-02T21:17:40,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-02T21:17:40,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8620 sec 2024-12-02T21:17:40,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.8670 sec 2024-12-02T21:17:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T21:17:41,069 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:42,179 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T21:17:49,069 INFO [Thread-1892 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1892. 2024-12-02T21:17:49,070 DEBUG [Thread-1892 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:49,070 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,070 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:49,171 INFO [Thread-1896 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1896. 2024-12-02T21:17:49,171 DEBUG [Thread-1896 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:49,171 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,171 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:49,180 INFO [Thread-1888 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1888. 2024-12-02T21:17:49,180 DEBUG [Thread-1888 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:49,180 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,180 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:49,190 INFO [Thread-1894 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1894. 2024-12-02T21:17:49,190 DEBUG [Thread-1894 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:17:49,190 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,190 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 218 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1667 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5001 rows 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1659 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4977 rows 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1669 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5001 rows 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1670 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5003 rows 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1666 2024-12-02T21:17:49,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4991 rows 2024-12-02T21:17:49,190 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:17:49,190 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:17:49,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:49,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:17:49,191 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T21:17:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T21:17:49,194 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174269194"}]},"ts":"1733174269194"} 2024-12-02T21:17:49,195 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T21:17:49,195 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T21:17:49,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T21:17:49,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62c2292167ae6a525d2d66119fcc23eb, UNASSIGN}] 2024-12-02T21:17:49,197 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62c2292167ae6a525d2d66119fcc23eb, UNASSIGN 2024-12-02T21:17:49,198 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=62c2292167ae6a525d2d66119fcc23eb, regionState=CLOSING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:49,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62c2292167ae6a525d2d66119fcc23eb, UNASSIGN because future has completed 2024-12-02T21:17:49,200 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:17:49,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:17:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T21:17:49,352 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 62c2292167ae6a525d2d66119fcc23eb, disabling compactions & flushes 2024-12-02T21:17:49,352 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. after waiting 0 ms 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:49,352 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 62c2292167ae6a525d2d66119fcc23eb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=A 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=B 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62c2292167ae6a525d2d66119fcc23eb, store=C 2024-12-02T21:17:49,352 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:49,356 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9aaab55444af462cb9a4b630116567a1 is 50, key is test_row_0/A:col10/1733174269189/Put/seqid=0 2024-12-02T21:17:49,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742318_1494 (size=12301) 2024-12-02T21:17:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T21:17:49,759 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9aaab55444af462cb9a4b630116567a1 2024-12-02T21:17:49,765 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/70967b889fa7486e8ba403ac1fc93137 is 50, key is test_row_0/B:col10/1733174269189/Put/seqid=0 2024-12-02T21:17:49,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742319_1495 (size=12301) 2024-12-02T21:17:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T21:17:50,169 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/70967b889fa7486e8ba403ac1fc93137 2024-12-02T21:17:50,175 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/5b76f01eb7b642a7b2cfddbdfdf99c06 is 50, key is test_row_0/C:col10/1733174269189/Put/seqid=0 2024-12-02T21:17:50,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742320_1496 (size=12301) 2024-12-02T21:17:50,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T21:17:50,579 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/5b76f01eb7b642a7b2cfddbdfdf99c06 2024-12-02T21:17:50,582 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/A/9aaab55444af462cb9a4b630116567a1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9aaab55444af462cb9a4b630116567a1 2024-12-02T21:17:50,585 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9aaab55444af462cb9a4b630116567a1, entries=150, sequenceid=331, filesize=12.0 K 2024-12-02T21:17:50,585 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/B/70967b889fa7486e8ba403ac1fc93137 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/70967b889fa7486e8ba403ac1fc93137 2024-12-02T21:17:50,588 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/70967b889fa7486e8ba403ac1fc93137, entries=150, sequenceid=331, filesize=12.0 K 2024-12-02T21:17:50,588 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/.tmp/C/5b76f01eb7b642a7b2cfddbdfdf99c06 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5b76f01eb7b642a7b2cfddbdfdf99c06 2024-12-02T21:17:50,590 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5b76f01eb7b642a7b2cfddbdfdf99c06, entries=150, sequenceid=331, filesize=12.0 K 2024-12-02T21:17:50,591 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 62c2292167ae6a525d2d66119fcc23eb in 1239ms, sequenceid=331, compaction requested=true 2024-12-02T21:17:50,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/09f4ffe7067c4d3ebee52b2c69f3c868, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/c06c79ace1e947188602e00c4edbd3bd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/e8f9809d1cee48b5bfb1c68e43fe4c57, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5ee3fd4a68b44521be4ae15feaa8524c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/3f2bb732878649b394fc5d79456e078c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9ff1251bf4b54cc6bb9a61f83bc80942, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0dd88167d8d4411b836faf6e99872958, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/70dc81f5bb40467281e844d36651f7ed, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/52ae3cd70d824948867c81cf07ecf79d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/af4e76bd53044e9a9a1dac45b3a3fdeb, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/d8c710425ff84f7cbc328838dd6a3b40, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/fa8c34f421764f7ca37b58f230a16422, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5bb19d4a47fa4f3396c6e67cedcb81ef, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/eb30dfa852fe47ac87e92bdcc297078a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0b58aad345b94e9590d13a554fbe5422, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9f5c4264afbc4e0993bc4cfa40131b26, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/ea32a3fca2594d55afde4bc999483f3e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/df1e525375f94a46b7e891ff636e2a2e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/f6a912a6c5354aff91948158594cb03e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/076e6ff7af2741edb2a1d4468581706c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/2b5c092baf3948fa96e0cb1d74bd2a8a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0e2f8e22c3da45fb8237be27d492f364, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9c8d7099519549978576608bd39b5c7d] to archive 2024-12-02T21:17:50,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:17:50,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/09f4ffe7067c4d3ebee52b2c69f3c868 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/09f4ffe7067c4d3ebee52b2c69f3c868 2024-12-02T21:17:50,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/c06c79ace1e947188602e00c4edbd3bd to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/c06c79ace1e947188602e00c4edbd3bd 2024-12-02T21:17:50,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/e8f9809d1cee48b5bfb1c68e43fe4c57 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/e8f9809d1cee48b5bfb1c68e43fe4c57 2024-12-02T21:17:50,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5ee3fd4a68b44521be4ae15feaa8524c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5ee3fd4a68b44521be4ae15feaa8524c 2024-12-02T21:17:50,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/3f2bb732878649b394fc5d79456e078c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/3f2bb732878649b394fc5d79456e078c 2024-12-02T21:17:50,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9ff1251bf4b54cc6bb9a61f83bc80942 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9ff1251bf4b54cc6bb9a61f83bc80942 2024-12-02T21:17:50,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0dd88167d8d4411b836faf6e99872958 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0dd88167d8d4411b836faf6e99872958 2024-12-02T21:17:50,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/70dc81f5bb40467281e844d36651f7ed to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/70dc81f5bb40467281e844d36651f7ed 2024-12-02T21:17:50,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/52ae3cd70d824948867c81cf07ecf79d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/52ae3cd70d824948867c81cf07ecf79d 2024-12-02T21:17:50,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/af4e76bd53044e9a9a1dac45b3a3fdeb to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/af4e76bd53044e9a9a1dac45b3a3fdeb 2024-12-02T21:17:50,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/d8c710425ff84f7cbc328838dd6a3b40 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/d8c710425ff84f7cbc328838dd6a3b40 2024-12-02T21:17:50,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/fa8c34f421764f7ca37b58f230a16422 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/fa8c34f421764f7ca37b58f230a16422 2024-12-02T21:17:50,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5bb19d4a47fa4f3396c6e67cedcb81ef to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/5bb19d4a47fa4f3396c6e67cedcb81ef 2024-12-02T21:17:50,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/eb30dfa852fe47ac87e92bdcc297078a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/eb30dfa852fe47ac87e92bdcc297078a 2024-12-02T21:17:50,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0b58aad345b94e9590d13a554fbe5422 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0b58aad345b94e9590d13a554fbe5422 2024-12-02T21:17:50,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9f5c4264afbc4e0993bc4cfa40131b26 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9f5c4264afbc4e0993bc4cfa40131b26 2024-12-02T21:17:50,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/ea32a3fca2594d55afde4bc999483f3e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/ea32a3fca2594d55afde4bc999483f3e 2024-12-02T21:17:50,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/df1e525375f94a46b7e891ff636e2a2e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/df1e525375f94a46b7e891ff636e2a2e 2024-12-02T21:17:50,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/f6a912a6c5354aff91948158594cb03e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/f6a912a6c5354aff91948158594cb03e 2024-12-02T21:17:50,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/076e6ff7af2741edb2a1d4468581706c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/076e6ff7af2741edb2a1d4468581706c 2024-12-02T21:17:50,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/2b5c092baf3948fa96e0cb1d74bd2a8a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/2b5c092baf3948fa96e0cb1d74bd2a8a 2024-12-02T21:17:50,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0e2f8e22c3da45fb8237be27d492f364 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/0e2f8e22c3da45fb8237be27d492f364 2024-12-02T21:17:50,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9c8d7099519549978576608bd39b5c7d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9c8d7099519549978576608bd39b5c7d 2024-12-02T21:17:50,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8640b2ca41194b4987df887be2532ab6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/9137fbe726e141b5ba84f2ffaffc271c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8bbba11abe5849c09aac87d103dbc7e0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/6d465b3f1ad34c2f8de254cf2e05c427, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/aa151c8e70d44c8eb45269be1dc4c073, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/1970d9b11917472499ebd2c8cfd02a49, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/0416ad719d6a4cd8ad66c52b8d321638, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f8b4c6e42aeb4a60a567ea99b631b7d9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/d6cd962f23d74382a550e92b6fbd0a23, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/bb29de8ea51f44609e69b3bb35d416ee, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/cc42d58f7e6e41a79def276135548bee, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/99f42cdb23e747839e908157d46d3c3d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f608d0b8c77a4c40b1dff2e3b51ce75d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/a31670ef5ffc4c93a79b56ff9a0abc7e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4ee96c0f47e54e5bb9834c5a405dfc66, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/49557926a7ac4e4bbbfbae1036585312, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/3348dac321e446819b49926640a0ed22, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4164482f8f64427380bc5847bc17c5a4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/570d5f96a9154b7da89eddeea1410e8b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/5705dcc8138442e2b8d3b9ddbbdbcdac, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/56bf4344b8694f019edd6420bf9ec023, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/51a229a8b42a47b2b7d02924baeca768, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8d103300da434b8b8fc41cf4178f0a09] to archive 2024-12-02T21:17:50,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:17:50,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8640b2ca41194b4987df887be2532ab6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8640b2ca41194b4987df887be2532ab6 2024-12-02T21:17:50,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/9137fbe726e141b5ba84f2ffaffc271c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/9137fbe726e141b5ba84f2ffaffc271c 2024-12-02T21:17:50,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8bbba11abe5849c09aac87d103dbc7e0 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8bbba11abe5849c09aac87d103dbc7e0 2024-12-02T21:17:50,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/6d465b3f1ad34c2f8de254cf2e05c427 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/6d465b3f1ad34c2f8de254cf2e05c427 2024-12-02T21:17:50,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/aa151c8e70d44c8eb45269be1dc4c073 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/aa151c8e70d44c8eb45269be1dc4c073 2024-12-02T21:17:50,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/1970d9b11917472499ebd2c8cfd02a49 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/1970d9b11917472499ebd2c8cfd02a49 2024-12-02T21:17:50,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/0416ad719d6a4cd8ad66c52b8d321638 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/0416ad719d6a4cd8ad66c52b8d321638 2024-12-02T21:17:50,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f8b4c6e42aeb4a60a567ea99b631b7d9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f8b4c6e42aeb4a60a567ea99b631b7d9 2024-12-02T21:17:50,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/d6cd962f23d74382a550e92b6fbd0a23 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/d6cd962f23d74382a550e92b6fbd0a23 2024-12-02T21:17:50,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/bb29de8ea51f44609e69b3bb35d416ee to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/bb29de8ea51f44609e69b3bb35d416ee 2024-12-02T21:17:50,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/cc42d58f7e6e41a79def276135548bee to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/cc42d58f7e6e41a79def276135548bee 2024-12-02T21:17:50,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/99f42cdb23e747839e908157d46d3c3d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/99f42cdb23e747839e908157d46d3c3d 2024-12-02T21:17:50,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f608d0b8c77a4c40b1dff2e3b51ce75d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/f608d0b8c77a4c40b1dff2e3b51ce75d 2024-12-02T21:17:50,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/a31670ef5ffc4c93a79b56ff9a0abc7e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/a31670ef5ffc4c93a79b56ff9a0abc7e 2024-12-02T21:17:50,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4ee96c0f47e54e5bb9834c5a405dfc66 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4ee96c0f47e54e5bb9834c5a405dfc66 2024-12-02T21:17:50,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/49557926a7ac4e4bbbfbae1036585312 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/49557926a7ac4e4bbbfbae1036585312 2024-12-02T21:17:50,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/3348dac321e446819b49926640a0ed22 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/3348dac321e446819b49926640a0ed22 2024-12-02T21:17:50,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4164482f8f64427380bc5847bc17c5a4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/4164482f8f64427380bc5847bc17c5a4 2024-12-02T21:17:50,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/570d5f96a9154b7da89eddeea1410e8b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/570d5f96a9154b7da89eddeea1410e8b 2024-12-02T21:17:50,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/5705dcc8138442e2b8d3b9ddbbdbcdac to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/5705dcc8138442e2b8d3b9ddbbdbcdac 2024-12-02T21:17:50,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/56bf4344b8694f019edd6420bf9ec023 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/56bf4344b8694f019edd6420bf9ec023 2024-12-02T21:17:50,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/51a229a8b42a47b2b7d02924baeca768 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/51a229a8b42a47b2b7d02924baeca768 2024-12-02T21:17:50,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8d103300da434b8b8fc41cf4178f0a09 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/8d103300da434b8b8fc41cf4178f0a09 2024-12-02T21:17:50,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/37773f69f41a47d0a545fb1d49c388ff, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c5ad07b87d4747899e593cb1e399b1e4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/012b9b2130cb402198a4c9c55637a7c5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/6352fa1292b64d598544d97d97b99aa0, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a39d8dad713b4ae5acf60a37c864c992, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/8579e69ea6e2421ebb98a455f23ebc96, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c177b7b216204e1bb62e993e333c25d7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5301089925d9403d8bfb48c5839c866f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/3e76e082be614e23b0cdca3295763238, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ec396c292b824da4808b18ec2c93fc6f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/14ed95052d4247919634a4721e6b2654, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/334ffd6552c449fdafd7d351b4beba6c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04595ab0081e4644930db55c3e0045b4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7da001c218c044e28227f9771ff9ed4c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a2170704b02d4f0894a311ee497565d9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/f126cb81eaf8445c8bd07160e2cec508, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/d61e6903792f4bd38f22f59c6cc27a1d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/cd6a7298989a47edb7725f6413f43342, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/223f9418244643159d442296078fc248, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ee88a23ac3d34172b594eb868b8169d1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7f5e4ee402ed441587705d24569a624b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7fdb3b4d50a5452b9b594c3006626883, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5576cc7a5ac14a5cbf77153c7f489386] to archive 2024-12-02T21:17:50,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:17:50,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/37773f69f41a47d0a545fb1d49c388ff to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/37773f69f41a47d0a545fb1d49c388ff 2024-12-02T21:17:50,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c5ad07b87d4747899e593cb1e399b1e4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c5ad07b87d4747899e593cb1e399b1e4 2024-12-02T21:17:50,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/012b9b2130cb402198a4c9c55637a7c5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/012b9b2130cb402198a4c9c55637a7c5 2024-12-02T21:17:50,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/6352fa1292b64d598544d97d97b99aa0 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/6352fa1292b64d598544d97d97b99aa0 2024-12-02T21:17:50,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a39d8dad713b4ae5acf60a37c864c992 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a39d8dad713b4ae5acf60a37c864c992 2024-12-02T21:17:50,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/8579e69ea6e2421ebb98a455f23ebc96 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/8579e69ea6e2421ebb98a455f23ebc96 2024-12-02T21:17:50,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c177b7b216204e1bb62e993e333c25d7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/c177b7b216204e1bb62e993e333c25d7 2024-12-02T21:17:50,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5301089925d9403d8bfb48c5839c866f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5301089925d9403d8bfb48c5839c866f 2024-12-02T21:17:50,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/3e76e082be614e23b0cdca3295763238 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/3e76e082be614e23b0cdca3295763238 2024-12-02T21:17:50,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ec396c292b824da4808b18ec2c93fc6f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ec396c292b824da4808b18ec2c93fc6f 2024-12-02T21:17:50,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/14ed95052d4247919634a4721e6b2654 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/14ed95052d4247919634a4721e6b2654 2024-12-02T21:17:50,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/334ffd6552c449fdafd7d351b4beba6c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/334ffd6552c449fdafd7d351b4beba6c 2024-12-02T21:17:50,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04595ab0081e4644930db55c3e0045b4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04595ab0081e4644930db55c3e0045b4 2024-12-02T21:17:50,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7da001c218c044e28227f9771ff9ed4c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7da001c218c044e28227f9771ff9ed4c 2024-12-02T21:17:50,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a2170704b02d4f0894a311ee497565d9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/a2170704b02d4f0894a311ee497565d9 2024-12-02T21:17:50,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/f126cb81eaf8445c8bd07160e2cec508 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/f126cb81eaf8445c8bd07160e2cec508 2024-12-02T21:17:50,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/d61e6903792f4bd38f22f59c6cc27a1d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/d61e6903792f4bd38f22f59c6cc27a1d 2024-12-02T21:17:50,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/cd6a7298989a47edb7725f6413f43342 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/cd6a7298989a47edb7725f6413f43342 2024-12-02T21:17:50,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/223f9418244643159d442296078fc248 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/223f9418244643159d442296078fc248 2024-12-02T21:17:50,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ee88a23ac3d34172b594eb868b8169d1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/ee88a23ac3d34172b594eb868b8169d1 2024-12-02T21:17:50,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7f5e4ee402ed441587705d24569a624b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7f5e4ee402ed441587705d24569a624b 2024-12-02T21:17:50,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7fdb3b4d50a5452b9b594c3006626883 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/7fdb3b4d50a5452b9b594c3006626883 2024-12-02T21:17:50,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5576cc7a5ac14a5cbf77153c7f489386 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5576cc7a5ac14a5cbf77153c7f489386 2024-12-02T21:17:50,644 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/recovered.edits/334.seqid, newMaxSeqId=334, maxSeqId=1 2024-12-02T21:17:50,644 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. 2024-12-02T21:17:50,644 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 62c2292167ae6a525d2d66119fcc23eb: Waiting for close lock at 1733174269352Running coprocessor pre-close hooks at 1733174269352Disabling compacts and flushes for region at 1733174269352Disabling writes for close at 1733174269352Obtaining lock to block concurrent updates at 1733174269352Preparing flush snapshotting stores in 62c2292167ae6a525d2d66119fcc23eb at 1733174269352Finished memstore snapshotting TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733174269353 (+1 ms)Flushing stores of TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb. at 1733174269353Flushing 62c2292167ae6a525d2d66119fcc23eb/A: creating writer at 1733174269353Flushing 62c2292167ae6a525d2d66119fcc23eb/A: appending metadata at 1733174269355 (+2 ms)Flushing 62c2292167ae6a525d2d66119fcc23eb/A: closing flushed file at 1733174269356 (+1 ms)Flushing 62c2292167ae6a525d2d66119fcc23eb/B: creating writer at 1733174269763 (+407 ms)Flushing 62c2292167ae6a525d2d66119fcc23eb/B: appending metadata at 1733174269765 (+2 ms)Flushing 62c2292167ae6a525d2d66119fcc23eb/B: closing flushed file at 1733174269765Flushing 62c2292167ae6a525d2d66119fcc23eb/C: creating writer at 1733174270173 (+408 ms)Flushing 62c2292167ae6a525d2d66119fcc23eb/C: appending metadata at 1733174270175 (+2 ms)Flushing 62c2292167ae6a525d2d66119fcc23eb/C: closing flushed file at 1733174270175Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5adfc443: reopening flushed file at 1733174270582 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b9d3894: reopening flushed file at 1733174270585 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13068334: reopening flushed file at 1733174270588 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 62c2292167ae6a525d2d66119fcc23eb in 1239ms, sequenceid=331, compaction requested=true at 1733174270591 (+3 ms)Writing region close event to WAL at 1733174270642 (+51 ms)Running coprocessor post-close hooks at 1733174270644 (+2 ms)Closed at 1733174270644 2024-12-02T21:17:50,646 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:50,646 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=62c2292167ae6a525d2d66119fcc23eb, regionState=CLOSED 2024-12-02T21:17:50,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:17:50,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-02T21:17:50,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 62c2292167ae6a525d2d66119fcc23eb, server=87c3fdb6c570,35563,1733174113859 in 1.4480 sec 2024-12-02T21:17:50,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-02T21:17:50,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62c2292167ae6a525d2d66119fcc23eb, UNASSIGN in 1.4520 sec 2024-12-02T21:17:50,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-02T21:17:50,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4550 sec 2024-12-02T21:17:50,653 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174270653"}]},"ts":"1733174270653"} 2024-12-02T21:17:50,654 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T21:17:50,654 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T21:17:50,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4640 sec 2024-12-02T21:17:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T21:17:51,339 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:51,340 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T21:17:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:51,341 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T21:17:51,342 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:51,343 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:51,345 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/recovered.edits] 2024-12-02T21:17:51,347 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/31bd6964648243d39f44bbaaeda72c6c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/31bd6964648243d39f44bbaaeda72c6c 2024-12-02T21:17:51,348 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9aaab55444af462cb9a4b630116567a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/9aaab55444af462cb9a4b630116567a1 2024-12-02T21:17:51,349 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/b3ee44901eaa4428abeebdd1566a9cb4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/A/b3ee44901eaa4428abeebdd1566a9cb4 2024-12-02T21:17:51,350 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/70967b889fa7486e8ba403ac1fc93137 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/70967b889fa7486e8ba403ac1fc93137 2024-12-02T21:17:51,351 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/74afed80d3e14c3ca635b8ae872dbdb0 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/74afed80d3e14c3ca635b8ae872dbdb0 2024-12-02T21:17:51,352 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/b62c5dcd8d0443e3bf630ee1b548c274 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/B/b62c5dcd8d0443e3bf630ee1b548c274 2024-12-02T21:17:51,353 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04191f517f07416ea7af3f6e80a1cab1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/04191f517f07416ea7af3f6e80a1cab1 2024-12-02T21:17:51,354 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/56c0e3cd35a1401f926a11421c479a8f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/56c0e3cd35a1401f926a11421c479a8f 2024-12-02T21:17:51,354 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5b76f01eb7b642a7b2cfddbdfdf99c06 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/C/5b76f01eb7b642a7b2cfddbdfdf99c06 2024-12-02T21:17:51,356 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/recovered.edits/334.seqid to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb/recovered.edits/334.seqid 2024-12-02T21:17:51,356 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/62c2292167ae6a525d2d66119fcc23eb 2024-12-02T21:17:51,356 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T21:17:51,358 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:51,360 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T21:17:51,362 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T21:17:51,363 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:51,363 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T21:17:51,363 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733174271363"}]},"ts":"9223372036854775807"} 2024-12-02T21:17:51,364 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T21:17:51,364 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 62c2292167ae6a525d2d66119fcc23eb, NAME => 'TestAcidGuarantees,,1733174237355.62c2292167ae6a525d2d66119fcc23eb.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T21:17:51,365 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T21:17:51,365 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733174271365"}]},"ts":"9223372036854775807"} 2024-12-02T21:17:51,366 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T21:17:51,367 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:51,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-12-02T21:17:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T21:17:51,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T21:17:51,451 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:51,456 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 248), OpenFileDescriptor=441 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=441 (was 462), ProcessCount=11 (was 11), AvailableMemoryMB=1957 (was 2395) 2024-12-02T21:17:51,463 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=441, ProcessCount=11, AvailableMemoryMB=1957 2024-12-02T21:17:51,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T21:17:51,465 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:17:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:51,466 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T21:17:51,466 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:51,466 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-02T21:17:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T21:17:51,467 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T21:17:51,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742321_1497 (size=1061) 2024-12-02T21:17:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T21:17:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T21:17:51,874 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0 2024-12-02T21:17:51,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742322_1498 (size=53) 2024-12-02T21:17:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T21:17:52,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:17:52,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6cc723b24523dcb8953ed05d209a77e3, disabling compactions & flushes 2024-12-02T21:17:52,282 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:52,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:52,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. after waiting 0 ms 2024-12-02T21:17:52,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:52,282 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:52,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6cc723b24523dcb8953ed05d209a77e3: Waiting for close lock at 1733174272282Disabling compacts and flushes for region at 1733174272282Disabling writes for close at 1733174272282Writing region close event to WAL at 1733174272282Closed at 1733174272282 2024-12-02T21:17:52,283 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T21:17:52,283 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733174272283"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174272283"}]},"ts":"1733174272283"} 2024-12-02T21:17:52,284 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T21:17:52,285 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T21:17:52,285 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174272285"}]},"ts":"1733174272285"} 2024-12-02T21:17:52,286 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T21:17:52,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, ASSIGN}] 2024-12-02T21:17:52,287 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, ASSIGN 2024-12-02T21:17:52,288 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, ASSIGN; state=OFFLINE, location=87c3fdb6c570,35563,1733174113859; forceNewPlan=false, retain=false 2024-12-02T21:17:52,438 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=6cc723b24523dcb8953ed05d209a77e3, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:52,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, ASSIGN because future has completed 2024-12-02T21:17:52,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:17:52,594 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:52,594 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:17:52,594 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,594 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:17:52,594 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,594 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,595 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,596 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:17:52,596 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6cc723b24523dcb8953ed05d209a77e3 columnFamilyName A 2024-12-02T21:17:52,596 DEBUG [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:52,597 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(327): Store=6cc723b24523dcb8953ed05d209a77e3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:17:52,597 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,598 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:17:52,598 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6cc723b24523dcb8953ed05d209a77e3 columnFamilyName B 2024-12-02T21:17:52,598 DEBUG [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:52,598 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(327): Store=6cc723b24523dcb8953ed05d209a77e3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:17:52,598 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,599 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:17:52,599 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6cc723b24523dcb8953ed05d209a77e3 columnFamilyName C 2024-12-02T21:17:52,599 DEBUG [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:52,600 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(327): Store=6cc723b24523dcb8953ed05d209a77e3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:17:52,600 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:52,600 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,600 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,601 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,602 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,602 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,602 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:17:52,603 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,604 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:17:52,605 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 6cc723b24523dcb8953ed05d209a77e3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59175062, jitterRate=-0.11822286248207092}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:17:52,605 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:52,605 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 6cc723b24523dcb8953ed05d209a77e3: Running coprocessor pre-open hook at 1733174272594Writing region info on filesystem at 1733174272594Initializing all the Stores at 1733174272595 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174272595Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174272595Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174272595Cleaning up temporary data from old regions at 1733174272602 (+7 ms)Running coprocessor post-open hooks at 1733174272605 (+3 ms)Region opened successfully at 1733174272605 2024-12-02T21:17:52,606 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., pid=144, masterSystemTime=1733174272591 2024-12-02T21:17:52,607 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:52,607 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:52,608 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=6cc723b24523dcb8953ed05d209a77e3, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T21:17:52,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:17:52,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-02T21:17:52,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 in 170 msec 2024-12-02T21:17:52,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-02T21:17:52,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, ASSIGN in 326 msec 2024-12-02T21:17:52,615 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T21:17:52,615 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174272615"}]},"ts":"1733174272615"} 2024-12-02T21:17:52,616 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T21:17:52,617 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T21:17:52,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-12-02T21:17:53,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T21:17:53,619 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:53,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:53,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:53,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:53,621 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a69879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:53,622 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc5516, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:53,623 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:53,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:53,625 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T21:17:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 87c3fdb6c570,39577,1733174113031 2024-12-02T21:17:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@34fd8581 2024-12-02T21:17:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T21:17:53,627 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T21:17:53,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T21:17:53,628 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:17:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T21:17:53,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742323_1499 (size=1097) 2024-12-02T21:17:54,063 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T21:17:54,063 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T21:17:54,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T21:17:54,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, REOPEN/MOVE}] 2024-12-02T21:17:54,068 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, REOPEN/MOVE 2024-12-02T21:17:54,069 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=6cc723b24523dcb8953ed05d209a77e3, regionState=CLOSING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, REOPEN/MOVE because future has completed 2024-12-02T21:17:54,070 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:17:54,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:17:54,222 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,222 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:17:54,222 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 6cc723b24523dcb8953ed05d209a77e3, disabling compactions & flushes 2024-12-02T21:17:54,222 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,222 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,222 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. after waiting 0 ms 2024-12-02T21:17:54,222 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,226 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:17:54,226 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,226 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 6cc723b24523dcb8953ed05d209a77e3: Waiting for close lock at 1733174274222Running coprocessor pre-close hooks at 1733174274222Disabling compacts and flushes for region at 1733174274222Disabling writes for close at 1733174274222Writing region close event to WAL at 1733174274223 (+1 ms)Running coprocessor post-close hooks at 1733174274226 (+3 ms)Closed at 1733174274226 2024-12-02T21:17:54,226 WARN [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 6cc723b24523dcb8953ed05d209a77e3 to self. 2024-12-02T21:17:54,228 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,229 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=6cc723b24523dcb8953ed05d209a77e3, regionState=CLOSED 2024-12-02T21:17:54,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:17:54,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-02T21:17:54,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 in 161 msec 2024-12-02T21:17:54,233 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, REOPEN/MOVE; state=CLOSED, location=87c3fdb6c570,35563,1733174113859; forceNewPlan=false, retain=true 2024-12-02T21:17:54,384 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=6cc723b24523dcb8953ed05d209a77e3, regionState=OPENING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, REOPEN/MOVE because future has completed 2024-12-02T21:17:54,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:17:54,539 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,539 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:17:54,539 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,539 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:17:54,540 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,540 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,540 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,541 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:17:54,541 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6cc723b24523dcb8953ed05d209a77e3 columnFamilyName A 2024-12-02T21:17:54,542 DEBUG [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:54,542 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(327): Store=6cc723b24523dcb8953ed05d209a77e3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:17:54,542 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,543 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:17:54,543 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6cc723b24523dcb8953ed05d209a77e3 columnFamilyName B 2024-12-02T21:17:54,543 DEBUG [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:54,543 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(327): Store=6cc723b24523dcb8953ed05d209a77e3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:17:54,543 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,544 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T21:17:54,544 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6cc723b24523dcb8953ed05d209a77e3 columnFamilyName C 2024-12-02T21:17:54,544 DEBUG [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:54,544 INFO [StoreOpener-6cc723b24523dcb8953ed05d209a77e3-1 {}] regionserver.HStore(327): Store=6cc723b24523dcb8953ed05d209a77e3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:17:54,544 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,545 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,545 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,546 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,547 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,547 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,547 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T21:17:54,548 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,549 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 6cc723b24523dcb8953ed05d209a77e3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70719431, jitterRate=0.05380164086818695}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T21:17:54,549 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,549 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 6cc723b24523dcb8953ed05d209a77e3: Running coprocessor pre-open hook at 1733174274540Writing region info on filesystem at 1733174274540Initializing all the Stores at 1733174274540Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174274540Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174274540Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174274540Cleaning up temporary data from old regions at 1733174274547 (+7 ms)Running coprocessor post-open hooks at 1733174274549 (+2 ms)Region opened successfully at 1733174274549 2024-12-02T21:17:54,550 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., pid=149, masterSystemTime=1733174274537 2024-12-02T21:17:54,551 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,551 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,552 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=6cc723b24523dcb8953ed05d209a77e3, regionState=OPEN, openSeqNum=5, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:17:54,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-02T21:17:54,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 in 169 msec 2024-12-02T21:17:54,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-02T21:17:54,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, REOPEN/MOVE in 487 msec 2024-12-02T21:17:54,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-02T21:17:54,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-12-02T21:17:54,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 931 msec 2024-12-02T21:17:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T21:17:54,561 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:54,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,562 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228bf6ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,564 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53319356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,566 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@731de1d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,567 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396730c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,569 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49bb3b8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,571 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5097190d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,572 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d50fda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,574 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2177ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,575 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e18ce86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,576 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74193094, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,578 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cb87a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177f6744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,580 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4338b9e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f05ce1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f417578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,585 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9db87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,587 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc5b426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2808adcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,39577,-1 for getting cluster id 2024-12-02T21:17:54,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:17:54,590 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1e0afb41-015e-46ad-ac26-3e94247c4bda' 2024-12-02T21:17:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:17:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1e0afb41-015e-46ad-ac26-3e94247c4bda" 2024-12-02T21:17:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f9d99c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,39577,-1] 2024-12-02T21:17:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:17:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:17:54,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:17:54,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e1b1269, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:17:54,592 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:54,592 DEBUG [Thread-2264 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,592 DEBUG [Thread-2260 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,592 DEBUG [Thread-2258 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,593 DEBUG [Thread-2249 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-02T21:17:54,593 DEBUG [Thread-2253 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,593 DEBUG [Thread-2256 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,593 DEBUG [Thread-2245 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,594 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T21:17:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,595 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:54,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:54,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,598 DEBUG [Thread-2262 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,599 DEBUG [Thread-2247 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,604 DEBUG [Thread-2251 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:17:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,35563,1733174113859, seqNum=-1] 2024-12-02T21:17:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:17:54,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:17:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:54,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T21:17:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5] 2024-12-02T21:17:54,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:17:54,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:54,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:17:54,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:54,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:17:54,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:54,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202040b4ae7bd474b27abe956a9ec29f892_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174274606/Put/seqid=0 2024-12-02T21:17:54,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742324_1500 (size=12154) 2024-12-02T21:17:54,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733174274624, max timestamp=1733174274660Num uniques 859; 2024-12-02T21:17:54,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T21:17:54,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733174274624, max timestamp=1733174274660Num uniques 859; 2024-12-02T21:17:54,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T21:17:54,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174274624, max timestamp=1733174274660Num uniques 859; 2024-12-02T21:17:54,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:17:54,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174334676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174334676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174334677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174334681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174334685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T21:17:54,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T21:17:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:54,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174334788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174334789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174334789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174334789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174334799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:54,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T21:17:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:54,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T21:17:55,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174334999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174334999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174334999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174334999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174335008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:55,052 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202040b4ae7bd474b27abe956a9ec29f892_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202040b4ae7bd474b27abe956a9ec29f892_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:55,054 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/0bd638990c2a47db947c19bd9f545437, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:17:55,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T21:17:55,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/0bd638990c2a47db947c19bd9f545437 is 175, key is test_row_0/A:col10/1733174274606/Put/seqid=0 2024-12-02T21:17:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:55,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742325_1501 (size=30955) 2024-12-02T21:17:55,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T21:17:55,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:55,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T21:17:55,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174335309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174335309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174335309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174335309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174335318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T21:17:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/0bd638990c2a47db947c19bd9f545437 2024-12-02T21:17:55,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/fa97b4586c8b4cf983ff890bc8a4af95 is 50, key is test_row_0/B:col10/1733174274606/Put/seqid=0 2024-12-02T21:17:55,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742326_1502 (size=12001) 2024-12-02T21:17:55,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/fa97b4586c8b4cf983ff890bc8a4af95 2024-12-02T21:17:55,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T21:17:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ccd989354d6e4f84b06053910690d012 is 50, key is test_row_0/C:col10/1733174274606/Put/seqid=0 2024-12-02T21:17:55,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742327_1503 (size=12001) 2024-12-02T21:17:55,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T21:17:55,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:55,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T21:17:55,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174335819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174335819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174335819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174335819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T21:17:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:55,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174335829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:55,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:55,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:55,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ccd989354d6e4f84b06053910690d012 2024-12-02T21:17:55,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/0bd638990c2a47db947c19bd9f545437 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/0bd638990c2a47db947c19bd9f545437 2024-12-02T21:17:55,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/0bd638990c2a47db947c19bd9f545437, entries=150, sequenceid=15, filesize=30.2 K 2024-12-02T21:17:55,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/fa97b4586c8b4cf983ff890bc8a4af95 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/fa97b4586c8b4cf983ff890bc8a4af95 2024-12-02T21:17:55,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/fa97b4586c8b4cf983ff890bc8a4af95, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T21:17:55,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ccd989354d6e4f84b06053910690d012 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ccd989354d6e4f84b06053910690d012 2024-12-02T21:17:55,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ccd989354d6e4f84b06053910690d012, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T21:17:55,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for 6cc723b24523dcb8953ed05d209a77e3 in 1357ms, sequenceid=15, compaction requested=false 2024-12-02T21:17:55,964 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T21:17:55,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:17:55,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T21:17:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:55,979 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=129.31 KB heapSize=369.23 KB 2024-12-02T21:17:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:17:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:17:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:17:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:17:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:17:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:17:55,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024fd0b8b2b0d64b128b1fbe68fe18596e_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174274679/Put/seqid=0 2024-12-02T21:17:55,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742328_1504 (size=12154) 2024-12-02T21:17:56,165 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T21:17:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:56,392 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024fd0b8b2b0d64b128b1fbe68fe18596e_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024fd0b8b2b0d64b128b1fbe68fe18596e_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:56,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/8db1cf9c05874f0eb1ecf0c01d36cbff, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:17:56,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/8db1cf9c05874f0eb1ecf0c01d36cbff is 175, key is test_row_0/A:col10/1733174274679/Put/seqid=0 2024-12-02T21:17:56,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742329_1505 (size=30955) 2024-12-02T21:17:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T21:17:56,796 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/8db1cf9c05874f0eb1ecf0c01d36cbff 2024-12-02T21:17:56,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/a989d6e06ca546e380330de726a1fa7a is 50, key is test_row_0/B:col10/1733174274679/Put/seqid=0 2024-12-02T21:17:56,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742330_1506 (size=12001) 2024-12-02T21:17:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:56,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:56,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174336844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174336845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174336846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:56,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174336848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:56,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174336849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:56,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174336958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174336959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174336959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174336959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174336959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174337169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174337169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174337169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174337169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174337171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,214 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/a989d6e06ca546e380330de726a1fa7a 2024-12-02T21:17:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/b52efdf85db64e5e8b0faf24cb0f105f is 50, key is test_row_0/C:col10/1733174274679/Put/seqid=0 2024-12-02T21:17:57,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742331_1507 (size=12001) 2024-12-02T21:17:57,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174337478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174337478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174337489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174337489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174337489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:57,666 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/b52efdf85db64e5e8b0faf24cb0f105f 2024-12-02T21:17:57,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/8db1cf9c05874f0eb1ecf0c01d36cbff as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/8db1cf9c05874f0eb1ecf0c01d36cbff 2024-12-02T21:17:57,672 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/8db1cf9c05874f0eb1ecf0c01d36cbff, entries=150, sequenceid=51, filesize=30.2 K 2024-12-02T21:17:57,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/a989d6e06ca546e380330de726a1fa7a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a989d6e06ca546e380330de726a1fa7a 2024-12-02T21:17:57,675 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a989d6e06ca546e380330de726a1fa7a, entries=150, sequenceid=51, filesize=11.7 K 2024-12-02T21:17:57,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/b52efdf85db64e5e8b0faf24cb0f105f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/b52efdf85db64e5e8b0faf24cb0f105f 2024-12-02T21:17:57,678 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/b52efdf85db64e5e8b0faf24cb0f105f, entries=150, sequenceid=51, filesize=11.7 K 2024-12-02T21:17:57,678 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132415, heapSize ~369.18 KB/378040, currentSize=60.38 KB/61830 for 6cc723b24523dcb8953ed05d209a77e3 in 1699ms, sequenceid=51, compaction requested=false 2024-12-02T21:17:57,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:17:57,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:57,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-02T21:17:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-02T21:17:57,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-02T21:17:57,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0850 sec 2024-12-02T21:17:57,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.0910 sec 2024-12-02T21:17:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:57,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T21:17:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:17:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:17:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:17:57,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:58,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202feb1a02c6bbe439ebffc21be33d7699a_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174277989/Put/seqid=0 2024-12-02T21:17:58,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742332_1508 (size=14594) 2024-12-02T21:17:58,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174338008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174338008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174338009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174338012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174338013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174338119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174338119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174338119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174338119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174338119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174338329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174338329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174338329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174338329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174338329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,410 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:58,413 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202feb1a02c6bbe439ebffc21be33d7699a_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202feb1a02c6bbe439ebffc21be33d7699a_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:58,413 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/7cda686370f74322b27e77b8509d8db6, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:17:58,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/7cda686370f74322b27e77b8509d8db6 is 175, key is test_row_0/A:col10/1733174277989/Put/seqid=0 2024-12-02T21:17:58,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742333_1509 (size=39549) 2024-12-02T21:17:58,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174338639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174338639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174338639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174338639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174338639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T21:17:58,749 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:17:58,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:17:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-02T21:17:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T21:17:58,751 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:17:58,751 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:17:58,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:17:58,824 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/7cda686370f74322b27e77b8509d8db6 2024-12-02T21:17:58,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/8e1d32a256dd4c2cb38ee69ab9f2f636 is 50, key is test_row_0/B:col10/1733174277989/Put/seqid=0 2024-12-02T21:17:58,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742334_1510 (size=12001) 2024-12-02T21:17:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T21:17:58,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T21:17:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:58,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:58,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T21:17:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T21:17:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174339149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174339149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174339149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174339149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:59,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:17:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174339149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:17:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:17:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:17:59,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T21:17:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/8e1d32a256dd4c2cb38ee69ab9f2f636 2024-12-02T21:17:59,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/9161787da3634bfabdb0b97cf4726304 is 50, key is test_row_0/C:col10/1733174277989/Put/seqid=0 2024-12-02T21:17:59,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742335_1511 (size=12001) 2024-12-02T21:17:59,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T21:17:59,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T21:17:59,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T21:17:59,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:59,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/9161787da3634bfabdb0b97cf4726304 2024-12-02T21:17:59,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/7cda686370f74322b27e77b8509d8db6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/7cda686370f74322b27e77b8509d8db6 2024-12-02T21:17:59,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/7cda686370f74322b27e77b8509d8db6, entries=200, sequenceid=65, filesize=38.6 K 2024-12-02T21:17:59,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/8e1d32a256dd4c2cb38ee69ab9f2f636 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8e1d32a256dd4c2cb38ee69ab9f2f636 2024-12-02T21:17:59,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8e1d32a256dd4c2cb38ee69ab9f2f636, entries=150, sequenceid=65, filesize=11.7 K 2024-12-02T21:17:59,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/9161787da3634bfabdb0b97cf4726304 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9161787da3634bfabdb0b97cf4726304 2024-12-02T21:17:59,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T21:17:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:17:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:17:59,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9161787da3634bfabdb0b97cf4726304, entries=150, sequenceid=65, filesize=11.7 K 2024-12-02T21:17:59,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 6cc723b24523dcb8953ed05d209a77e3 in 1685ms, sequenceid=65, compaction requested=true 2024-12-02T21:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:59,675 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:17:59,675 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:59,677 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:59,677 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:59,677 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/A is initiating minor compaction (all files) 2024-12-02T21:17:59,677 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/B is initiating minor compaction (all files) 2024-12-02T21:17:59,677 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/A in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,677 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/B in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,677 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/0bd638990c2a47db947c19bd9f545437, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/8db1cf9c05874f0eb1ecf0c01d36cbff, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/7cda686370f74322b27e77b8509d8db6] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=99.1 K 2024-12-02T21:17:59,677 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/fa97b4586c8b4cf983ff890bc8a4af95, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a989d6e06ca546e380330de726a1fa7a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8e1d32a256dd4c2cb38ee69ab9f2f636] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=35.2 K 2024-12-02T21:17:59,677 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,677 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/0bd638990c2a47db947c19bd9f545437, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/8db1cf9c05874f0eb1ecf0c01d36cbff, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/7cda686370f74322b27e77b8509d8db6] 2024-12-02T21:17:59,678 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting fa97b4586c8b4cf983ff890bc8a4af95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733174274600 2024-12-02T21:17:59,678 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd638990c2a47db947c19bd9f545437, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733174274600 2024-12-02T21:17:59,679 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting a989d6e06ca546e380330de726a1fa7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733174274673 2024-12-02T21:17:59,679 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e1d32a256dd4c2cb38ee69ab9f2f636, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733174276843 2024-12-02T21:17:59,679 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8db1cf9c05874f0eb1ecf0c01d36cbff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733174274673 2024-12-02T21:17:59,680 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cda686370f74322b27e77b8509d8db6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733174276843 2024-12-02T21:17:59,698 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:59,698 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/5530a4537e464b1299f9ed41d122c35b is 50, key is test_row_0/B:col10/1733174277989/Put/seqid=0 2024-12-02T21:17:59,707 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:17:59,729 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202496e475576de444dbb9a81ef4d243467_6cc723b24523dcb8953ed05d209a77e3 store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:17:59,731 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202496e475576de444dbb9a81ef4d243467_6cc723b24523dcb8953ed05d209a77e3, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:17:59,731 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202496e475576de444dbb9a81ef4d243467_6cc723b24523dcb8953ed05d209a77e3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:17:59,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742336_1512 (size=12104) 2024-12-02T21:17:59,753 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/5530a4537e464b1299f9ed41d122c35b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/5530a4537e464b1299f9ed41d122c35b 2024-12-02T21:17:59,759 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/B of 6cc723b24523dcb8953ed05d209a77e3 into 5530a4537e464b1299f9ed41d122c35b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:59,759 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:17:59,759 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/B, priority=13, startTime=1733174279675; duration=0sec 2024-12-02T21:17:59,759 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:17:59,759 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:B 2024-12-02T21:17:59,759 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:17:59,761 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:17:59,761 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/C is initiating minor compaction (all files) 2024-12-02T21:17:59,761 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/C in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,761 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ccd989354d6e4f84b06053910690d012, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/b52efdf85db64e5e8b0faf24cb0f105f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9161787da3634bfabdb0b97cf4726304] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=35.2 K 2024-12-02T21:17:59,761 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd989354d6e4f84b06053910690d012, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733174274600 2024-12-02T21:17:59,762 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b52efdf85db64e5e8b0faf24cb0f105f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733174274673 2024-12-02T21:17:59,762 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 9161787da3634bfabdb0b97cf4726304, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733174276843 2024-12-02T21:17:59,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742337_1513 (size=4469) 2024-12-02T21:17:59,770 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:59,770 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/48d261fb365d426393b5bed4f4271e7f is 50, key is test_row_0/C:col10/1733174277989/Put/seqid=0 2024-12-02T21:17:59,771 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#A#compaction#435 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:17:59,771 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/71a7eed705ac454db6feb8d8435fa632 is 175, key is test_row_0/A:col10/1733174277989/Put/seqid=0 2024-12-02T21:17:59,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742338_1514 (size=31058) 2024-12-02T21:17:59,808 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/71a7eed705ac454db6feb8d8435fa632 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71a7eed705ac454db6feb8d8435fa632 2024-12-02T21:17:59,813 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/A of 6cc723b24523dcb8953ed05d209a77e3 into 71a7eed705ac454db6feb8d8435fa632(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:59,814 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:17:59,814 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/A, priority=13, startTime=1733174279675; duration=0sec 2024-12-02T21:17:59,814 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:59,814 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:A 2024-12-02T21:17:59,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742339_1515 (size=12104) 2024-12-02T21:17:59,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T21:17:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:17:59,820 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T21:17:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:17:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:17:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:17:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:17:59,829 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/48d261fb365d426393b5bed4f4271e7f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/48d261fb365d426393b5bed4f4271e7f 2024-12-02T21:17:59,833 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/C of 6cc723b24523dcb8953ed05d209a77e3 into 48d261fb365d426393b5bed4f4271e7f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:17:59,834 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:17:59,834 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/C, priority=13, startTime=1733174279675; duration=0sec 2024-12-02T21:17:59,834 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:17:59,834 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:C 2024-12-02T21:17:59,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202061ffcd2329a4e64929ef19c259a5a8a_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174278006/Put/seqid=0 2024-12-02T21:17:59,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742340_1516 (size=12154) 2024-12-02T21:17:59,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:17:59,871 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202061ffcd2329a4e64929ef19c259a5a8a_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202061ffcd2329a4e64929ef19c259a5a8a_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:17:59,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/938980e52da648488a3b7f2e2aa7b172, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:17:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/938980e52da648488a3b7f2e2aa7b172 is 175, key is test_row_0/A:col10/1733174278006/Put/seqid=0 2024-12-02T21:17:59,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742341_1517 (size=30955) 2024-12-02T21:17:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T21:18:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:00,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174340170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174340172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174340174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174340174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174340174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174340279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174340279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174340279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,282 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/938980e52da648488a3b7f2e2aa7b172 2024-12-02T21:18:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/cfb02b910b8a4e498ec03788d3708a52 is 50, key is test_row_0/B:col10/1733174278006/Put/seqid=0 2024-12-02T21:18:00,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174340288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742342_1518 (size=12001) 2024-12-02T21:18:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174340288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,291 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/cfb02b910b8a4e498ec03788d3708a52 2024-12-02T21:18:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/5d0ebde9bf67432a99fecfd5aeb54df1 is 50, key is test_row_0/C:col10/1733174278006/Put/seqid=0 2024-12-02T21:18:00,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742343_1519 (size=12001) 2024-12-02T21:18:00,301 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/5d0ebde9bf67432a99fecfd5aeb54df1 2024-12-02T21:18:00,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/938980e52da648488a3b7f2e2aa7b172 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/938980e52da648488a3b7f2e2aa7b172 2024-12-02T21:18:00,307 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/938980e52da648488a3b7f2e2aa7b172, entries=150, sequenceid=86, filesize=30.2 K 2024-12-02T21:18:00,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/cfb02b910b8a4e498ec03788d3708a52 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/cfb02b910b8a4e498ec03788d3708a52 2024-12-02T21:18:00,313 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/cfb02b910b8a4e498ec03788d3708a52, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T21:18:00,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/5d0ebde9bf67432a99fecfd5aeb54df1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5d0ebde9bf67432a99fecfd5aeb54df1 2024-12-02T21:18:00,317 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5d0ebde9bf67432a99fecfd5aeb54df1, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T21:18:00,318 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6cc723b24523dcb8953ed05d209a77e3 in 498ms, sequenceid=86, compaction requested=false 2024-12-02T21:18:00,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:00,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:00,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-02T21:18:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-02T21:18:00,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-02T21:18:00,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-12-02T21:18:00,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.5730 sec 2024-12-02T21:18:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:00,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:18:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:00,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:00,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:00,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:00,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028aa3ded80f9c4bd3ab56114d1050ec10_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174280492/Put/seqid=0 2024-12-02T21:18:00,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742344_1520 (size=12154) 2024-12-02T21:18:00,510 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:00,513 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028aa3ded80f9c4bd3ab56114d1050ec10_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028aa3ded80f9c4bd3ab56114d1050ec10_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:00,513 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/a3428e8fcb89464782abecf0f28aa936, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:00,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/a3428e8fcb89464782abecf0f28aa936 is 175, key is test_row_0/A:col10/1733174280492/Put/seqid=0 2024-12-02T21:18:00,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174340509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174340510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174340511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174340516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174340516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742345_1521 (size=30955) 2024-12-02T21:18:00,571 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/a3428e8fcb89464782abecf0f28aa936 2024-12-02T21:18:00,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/95a6ac8c921f468a9283d7e309f97d2e is 50, key is test_row_0/B:col10/1733174280492/Put/seqid=0 2024-12-02T21:18:00,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742346_1522 (size=12001) 2024-12-02T21:18:00,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/95a6ac8c921f468a9283d7e309f97d2e 2024-12-02T21:18:00,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/a2e99ff9af4f47b1a2d7940009e2c587 is 50, key is test_row_0/C:col10/1733174280492/Put/seqid=0 2024-12-02T21:18:00,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742347_1523 (size=12001) 2024-12-02T21:18:00,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174340618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174340618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174340619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174340629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174340629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174340829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174340829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174340829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174340838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174340838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T21:18:00,898 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:00,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:18:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-02T21:18:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T21:18:00,901 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:18:00,901 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:18:00,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:18:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T21:18:01,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/a2e99ff9af4f47b1a2d7940009e2c587 2024-12-02T21:18:01,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/a3428e8fcb89464782abecf0f28aa936 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/a3428e8fcb89464782abecf0f28aa936 2024-12-02T21:18:01,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/a3428e8fcb89464782abecf0f28aa936, entries=150, sequenceid=103, filesize=30.2 K 2024-12-02T21:18:01,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/95a6ac8c921f468a9283d7e309f97d2e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/95a6ac8c921f468a9283d7e309f97d2e 2024-12-02T21:18:01,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/95a6ac8c921f468a9283d7e309f97d2e, entries=150, sequenceid=103, filesize=11.7 K 2024-12-02T21:18:01,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/a2e99ff9af4f47b1a2d7940009e2c587 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/a2e99ff9af4f47b1a2d7940009e2c587 2024-12-02T21:18:01,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/a2e99ff9af4f47b1a2d7940009e2c587, entries=150, sequenceid=103, filesize=11.7 K 2024-12-02T21:18:01,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 6cc723b24523dcb8953ed05d209a77e3 in 534ms, sequenceid=103, compaction requested=true 2024-12-02T21:18:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:18:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:01,026 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:18:01,026 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:18:01,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:18:01,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:01,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:18:01,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:18:01,027 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:18:01,027 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/B is initiating minor compaction (all files) 2024-12-02T21:18:01,027 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/B in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:01,027 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/5530a4537e464b1299f9ed41d122c35b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/cfb02b910b8a4e498ec03788d3708a52, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/95a6ac8c921f468a9283d7e309f97d2e] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=35.3 K 2024-12-02T21:18:01,028 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:18:01,028 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/A is initiating minor compaction (all files) 2024-12-02T21:18:01,028 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/A in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:01,028 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71a7eed705ac454db6feb8d8435fa632, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/938980e52da648488a3b7f2e2aa7b172, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/a3428e8fcb89464782abecf0f28aa936] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=90.8 K 2024-12-02T21:18:01,028 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:01,028 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71a7eed705ac454db6feb8d8435fa632, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/938980e52da648488a3b7f2e2aa7b172, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/a3428e8fcb89464782abecf0f28aa936] 2024-12-02T21:18:01,028 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5530a4537e464b1299f9ed41d122c35b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733174276843 2024-12-02T21:18:01,028 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71a7eed705ac454db6feb8d8435fa632, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733174276843 2024-12-02T21:18:01,028 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938980e52da648488a3b7f2e2aa7b172, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733174278006 2024-12-02T21:18:01,028 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb02b910b8a4e498ec03788d3708a52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733174278006 2024-12-02T21:18:01,029 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 95a6ac8c921f468a9283d7e309f97d2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733174280172 2024-12-02T21:18:01,029 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3428e8fcb89464782abecf0f28aa936, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733174280172 2024-12-02T21:18:01,049 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:01,050 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#B#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:01,050 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/f9e77bb21a33458194e1a5edd183cd10 is 50, key is test_row_0/B:col10/1733174280492/Put/seqid=0 2024-12-02T21:18:01,051 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120287727e9922a34c37808195c0ee12b9f6_6cc723b24523dcb8953ed05d209a77e3 store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:01,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T21:18:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:01,053 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120287727e9922a34c37808195c0ee12b9f6_6cc723b24523dcb8953ed05d209a77e3, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:01,053 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T21:18:01,053 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120287727e9922a34c37808195c0ee12b9f6_6cc723b24523dcb8953ed05d209a77e3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:01,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742348_1524 (size=12207) 2024-12-02T21:18:01,085 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/f9e77bb21a33458194e1a5edd183cd10 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/f9e77bb21a33458194e1a5edd183cd10 2024-12-02T21:18:01,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742349_1525 (size=4469) 2024-12-02T21:18:01,091 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/B of 6cc723b24523dcb8953ed05d209a77e3 into f9e77bb21a33458194e1a5edd183cd10(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:01,091 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:01,091 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/B, priority=13, startTime=1733174281026; duration=0sec 2024-12-02T21:18:01,091 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:18:01,091 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:B 2024-12-02T21:18:01,091 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:18:01,092 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#A#compaction#444 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:01,092 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/4b29a27b71f446599eec33c5175f4b19 is 175, key is test_row_0/A:col10/1733174280492/Put/seqid=0 2024-12-02T21:18:01,093 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:18:01,093 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/C is initiating minor compaction (all files) 2024-12-02T21:18:01,093 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/C in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:01,093 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/48d261fb365d426393b5bed4f4271e7f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5d0ebde9bf67432a99fecfd5aeb54df1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/a2e99ff9af4f47b1a2d7940009e2c587] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=35.3 K 2024-12-02T21:18:01,094 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d261fb365d426393b5bed4f4271e7f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733174276843 2024-12-02T21:18:01,094 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d0ebde9bf67432a99fecfd5aeb54df1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733174278006 2024-12-02T21:18:01,095 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e99ff9af4f47b1a2d7940009e2c587, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733174280172 2024-12-02T21:18:01,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120284b91adb9621494abea5f55190572b07_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174280507/Put/seqid=0 2024-12-02T21:18:01,106 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:01,107 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/00a76cd790a949f4935185cf93577ae4 is 50, key is test_row_0/C:col10/1733174280492/Put/seqid=0 2024-12-02T21:18:01,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742350_1526 (size=31161) 2024-12-02T21:18:01,133 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/4b29a27b71f446599eec33c5175f4b19 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/4b29a27b71f446599eec33c5175f4b19 2024-12-02T21:18:01,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742352_1528 (size=12207) 2024-12-02T21:18:01,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742351_1527 (size=12154) 2024-12-02T21:18:01,139 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/A of 6cc723b24523dcb8953ed05d209a77e3 into 4b29a27b71f446599eec33c5175f4b19(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:01,140 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:01,140 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/A, priority=13, startTime=1733174281026; duration=0sec 2024-12-02T21:18:01,140 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:01,140 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:A 2024-12-02T21:18:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:01,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:01,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174341151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174341152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174341152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174341153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174341155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T21:18:01,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174341258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174341268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174341269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174341269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174341269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174341468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174341478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174341478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174341479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174341479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T21:18:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:01,539 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120284b91adb9621494abea5f55190572b07_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120284b91adb9621494abea5f55190572b07_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:01,539 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/00a76cd790a949f4935185cf93577ae4 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/00a76cd790a949f4935185cf93577ae4 2024-12-02T21:18:01,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/556c7667c4824c7592aec82f6668ece3, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:01,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/556c7667c4824c7592aec82f6668ece3 is 175, key is test_row_0/A:col10/1733174280507/Put/seqid=0 2024-12-02T21:18:01,544 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/C of 6cc723b24523dcb8953ed05d209a77e3 into 00a76cd790a949f4935185cf93577ae4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:01,544 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:01,544 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/C, priority=13, startTime=1733174281027; duration=0sec 2024-12-02T21:18:01,544 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:01,544 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:C 2024-12-02T21:18:01,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742353_1529 (size=30955) 2024-12-02T21:18:01,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174341779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174341788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174341788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174341788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174341789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:01,950 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/556c7667c4824c7592aec82f6668ece3 2024-12-02T21:18:01,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/8feeb01d9c2e4ee893cf38f3c61df395 is 50, key is test_row_0/B:col10/1733174280507/Put/seqid=0 2024-12-02T21:18:01,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742354_1530 (size=12001) 2024-12-02T21:18:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T21:18:02,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174342289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:02,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174342298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:02,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174342298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:02,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174342298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174342298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:02,359 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/8feeb01d9c2e4ee893cf38f3c61df395 2024-12-02T21:18:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/58e1b7cc3c92475a8863c8ba751cf3cf is 50, key is test_row_0/C:col10/1733174280507/Put/seqid=0 2024-12-02T21:18:02,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742355_1531 (size=12001) 2024-12-02T21:18:02,368 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/58e1b7cc3c92475a8863c8ba751cf3cf 2024-12-02T21:18:02,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/556c7667c4824c7592aec82f6668ece3 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/556c7667c4824c7592aec82f6668ece3 2024-12-02T21:18:02,375 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/556c7667c4824c7592aec82f6668ece3, entries=150, sequenceid=120, filesize=30.2 K 2024-12-02T21:18:02,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/8feeb01d9c2e4ee893cf38f3c61df395 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8feeb01d9c2e4ee893cf38f3c61df395 2024-12-02T21:18:02,379 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8feeb01d9c2e4ee893cf38f3c61df395, entries=150, sequenceid=120, filesize=11.7 K 2024-12-02T21:18:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/58e1b7cc3c92475a8863c8ba751cf3cf as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/58e1b7cc3c92475a8863c8ba751cf3cf 2024-12-02T21:18:02,382 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/58e1b7cc3c92475a8863c8ba751cf3cf, entries=150, sequenceid=120, filesize=11.7 K 2024-12-02T21:18:02,383 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6cc723b24523dcb8953ed05d209a77e3 in 1330ms, sequenceid=120, compaction requested=false 2024-12-02T21:18:02,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:02,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:02,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-02T21:18:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-02T21:18:02,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-02T21:18:02,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4830 sec 2024-12-02T21:18:02,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4870 sec 2024-12-02T21:18:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T21:18:03,059 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:03,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:18:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-02T21:18:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T21:18:03,061 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:18:03,061 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:18:03,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:18:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T21:18:03,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T21:18:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:03,214 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T21:18:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bf5f7a1f5a724067ba9882a0f754678b_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174281150/Put/seqid=0 2024-12-02T21:18:03,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742356_1532 (size=12304) 2024-12-02T21:18:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:03,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174343319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174343320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174343320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174343321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174343321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T21:18:03,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174343429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174343429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174343429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174343430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174343430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:03,627 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bf5f7a1f5a724067ba9882a0f754678b_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bf5f7a1f5a724067ba9882a0f754678b_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/fc1f5bec9f29498495259d93728b15fd, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:03,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/fc1f5bec9f29498495259d93728b15fd is 175, key is test_row_0/A:col10/1733174281150/Put/seqid=0 2024-12-02T21:18:03,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742357_1533 (size=31105) 2024-12-02T21:18:03,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174343638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174343638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174343639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174343639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174343639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T21:18:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174343949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174343949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174343949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174343949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:03,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174343949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:04,031 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/fc1f5bec9f29498495259d93728b15fd 2024-12-02T21:18:04,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/72a08964c05045f3bd0b453cd79b43f5 is 50, key is test_row_0/B:col10/1733174281150/Put/seqid=0 2024-12-02T21:18:04,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742358_1534 (size=12151) 2024-12-02T21:18:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T21:18:04,441 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/72a08964c05045f3bd0b453cd79b43f5 2024-12-02T21:18:04,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174344459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:04,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174344459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174344459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:04,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174344459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:04,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174344459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:04,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/41db991a8e674d65a56638652e6d5d3a is 50, key is test_row_0/C:col10/1733174281150/Put/seqid=0 2024-12-02T21:18:04,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742359_1535 (size=12151) 2024-12-02T21:18:04,556 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/41db991a8e674d65a56638652e6d5d3a 2024-12-02T21:18:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/fc1f5bec9f29498495259d93728b15fd as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/fc1f5bec9f29498495259d93728b15fd 2024-12-02T21:18:04,562 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/fc1f5bec9f29498495259d93728b15fd, entries=150, sequenceid=138, filesize=30.4 K 2024-12-02T21:18:04,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/72a08964c05045f3bd0b453cd79b43f5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/72a08964c05045f3bd0b453cd79b43f5 2024-12-02T21:18:04,565 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/72a08964c05045f3bd0b453cd79b43f5, entries=150, sequenceid=138, filesize=11.9 K 2024-12-02T21:18:04,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/41db991a8e674d65a56638652e6d5d3a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/41db991a8e674d65a56638652e6d5d3a 2024-12-02T21:18:04,570 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/41db991a8e674d65a56638652e6d5d3a, entries=150, sequenceid=138, filesize=11.9 K 2024-12-02T21:18:04,570 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6cc723b24523dcb8953ed05d209a77e3 in 1356ms, sequenceid=138, compaction requested=true 2024-12-02T21:18:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:04,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:04,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-02T21:18:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-02T21:18:04,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-02T21:18:04,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5120 sec 2024-12-02T21:18:04,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5160 sec 2024-12-02T21:18:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T21:18:05,209 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:05,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:18:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-02T21:18:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T21:18:05,212 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:18:05,212 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:18:05,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:18:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T21:18:05,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T21:18:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:05,363 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T21:18:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:05,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025ec0d57eaab64a818ad47fd702b462fc_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174283318/Put/seqid=0 2024-12-02T21:18:05,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742360_1536 (size=12304) 2024-12-02T21:18:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:05,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174345480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174345480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174345483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174345484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174345484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T21:18:05,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174345589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174345589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174345599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174345599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174345599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:05,784 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025ec0d57eaab64a818ad47fd702b462fc_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025ec0d57eaab64a818ad47fd702b462fc_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:05,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/6af085a7dcdf488f859f4a7e2d46afe7, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:05,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/6af085a7dcdf488f859f4a7e2d46afe7 is 175, key is test_row_0/A:col10/1733174283318/Put/seqid=0 2024-12-02T21:18:05,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742361_1537 (size=31105) 2024-12-02T21:18:05,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174345798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174345798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174345809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174345809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174345809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T21:18:06,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174346108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174346108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174346119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174346119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174346119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,195 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/6af085a7dcdf488f859f4a7e2d46afe7 2024-12-02T21:18:06,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/b804a4407cf747499718f2fa68c1ee8c is 50, key is test_row_0/B:col10/1733174283318/Put/seqid=0 2024-12-02T21:18:06,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742362_1538 (size=12151) 2024-12-02T21:18:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T21:18:06,604 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/b804a4407cf747499718f2fa68c1ee8c 2024-12-02T21:18:06,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/85bd4ea8b4724533883d7341c4388efe is 50, key is test_row_0/C:col10/1733174283318/Put/seqid=0 2024-12-02T21:18:06,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742363_1539 (size=12151) 2024-12-02T21:18:06,618 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/85bd4ea8b4724533883d7341c4388efe 2024-12-02T21:18:06,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174346618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174346618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/6af085a7dcdf488f859f4a7e2d46afe7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6af085a7dcdf488f859f4a7e2d46afe7 2024-12-02T21:18:06,627 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6af085a7dcdf488f859f4a7e2d46afe7, entries=150, sequenceid=155, filesize=30.4 K 2024-12-02T21:18:06,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/b804a4407cf747499718f2fa68c1ee8c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b804a4407cf747499718f2fa68c1ee8c 2024-12-02T21:18:06,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174346629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174346629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174346629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:06,632 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b804a4407cf747499718f2fa68c1ee8c, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T21:18:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/85bd4ea8b4724533883d7341c4388efe as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/85bd4ea8b4724533883d7341c4388efe 2024-12-02T21:18:06,637 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/85bd4ea8b4724533883d7341c4388efe, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T21:18:06,638 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6cc723b24523dcb8953ed05d209a77e3 in 1275ms, sequenceid=155, compaction requested=true 2024-12-02T21:18:06,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:06,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:06,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-02T21:18:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-02T21:18:06,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-02T21:18:06,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4280 sec 2024-12-02T21:18:06,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4340 sec 2024-12-02T21:18:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T21:18:07,359 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:07,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:18:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-02T21:18:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T21:18:07,361 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:18:07,361 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:18:07,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:18:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T21:18:07,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T21:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:07,513 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T21:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:07,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021c11d088a657402f8222c57355c271cf_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174285474/Put/seqid=0 2024-12-02T21:18:07,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742364_1540 (size=12304) 2024-12-02T21:18:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:07,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174347668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174347671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174347672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174347672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174347673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T21:18:07,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174347778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174347778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174347789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174347789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174347789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:07,926 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021c11d088a657402f8222c57355c271cf_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021c11d088a657402f8222c57355c271cf_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/858f8b9686a84e3f9ac6dbd3e0e88821, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/858f8b9686a84e3f9ac6dbd3e0e88821 is 175, key is test_row_0/A:col10/1733174285474/Put/seqid=0 2024-12-02T21:18:07,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742365_1541 (size=31105) 2024-12-02T21:18:07,932 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/858f8b9686a84e3f9ac6dbd3e0e88821 2024-12-02T21:18:07,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/b611c3ab2246444eaff0cd0116dfc079 is 50, key is test_row_0/B:col10/1733174285474/Put/seqid=0 2024-12-02T21:18:07,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742366_1542 (size=12151) 2024-12-02T21:18:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T21:18:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174347988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174347988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174347999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174347999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174347999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174348298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174348298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174348308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174348309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174348309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,351 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/b611c3ab2246444eaff0cd0116dfc079 2024-12-02T21:18:08,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ac9d39388569473ab60ab3886b2f0e7f is 50, key is test_row_0/C:col10/1733174285474/Put/seqid=0 2024-12-02T21:18:08,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742367_1543 (size=12151) 2024-12-02T21:18:08,363 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ac9d39388569473ab60ab3886b2f0e7f 2024-12-02T21:18:08,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/858f8b9686a84e3f9ac6dbd3e0e88821 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/858f8b9686a84e3f9ac6dbd3e0e88821 2024-12-02T21:18:08,371 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/858f8b9686a84e3f9ac6dbd3e0e88821, entries=150, sequenceid=170, filesize=30.4 K 2024-12-02T21:18:08,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/b611c3ab2246444eaff0cd0116dfc079 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b611c3ab2246444eaff0cd0116dfc079 2024-12-02T21:18:08,375 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b611c3ab2246444eaff0cd0116dfc079, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T21:18:08,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ac9d39388569473ab60ab3886b2f0e7f as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac9d39388569473ab60ab3886b2f0e7f 2024-12-02T21:18:08,377 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac9d39388569473ab60ab3886b2f0e7f, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T21:18:08,378 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6cc723b24523dcb8953ed05d209a77e3 in 865ms, sequenceid=170, compaction requested=true 2024-12-02T21:18:08,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:08,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:08,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-02T21:18:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-02T21:18:08,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-02T21:18:08,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-12-02T21:18:08,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.0230 sec 2024-12-02T21:18:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T21:18:08,499 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:08,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:18:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-02T21:18:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T21:18:08,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:18:08,502 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:18:08,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:18:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T21:18:08,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T21:18:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:08,655 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T21:18:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:08,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120209d502f35e454ec0ac082de4dd98c635_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174287670/Put/seqid=0 2024-12-02T21:18:08,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742368_1544 (size=12304) 2024-12-02T21:18:08,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:08,671 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120209d502f35e454ec0ac082de4dd98c635_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120209d502f35e454ec0ac082de4dd98c635_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/9db5cb55d57244689c48b47cae55c52c, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:08,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/9db5cb55d57244689c48b47cae55c52c is 175, key is test_row_0/A:col10/1733174287670/Put/seqid=0 2024-12-02T21:18:08,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742369_1545 (size=31105) 2024-12-02T21:18:08,677 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/9db5cb55d57244689c48b47cae55c52c 2024-12-02T21:18:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/386c5469af71438bad57cb72b16322d3 is 50, key is test_row_0/B:col10/1733174287670/Put/seqid=0 2024-12-02T21:18:08,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742370_1546 (size=12151) 2024-12-02T21:18:08,687 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/386c5469af71438bad57cb72b16322d3 2024-12-02T21:18:08,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/9729612020fe4e65b06290c687ecf581 is 50, key is test_row_0/C:col10/1733174287670/Put/seqid=0 2024-12-02T21:18:08,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742371_1547 (size=12151) 2024-12-02T21:18:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:08,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T21:18:08,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174348826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174348826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174348827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174348833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174348834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174348939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174348939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174348939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174348948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:08,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174348949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:08,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:08,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,101 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/9729612020fe4e65b06290c687ecf581 2024-12-02T21:18:09,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/9db5cb55d57244689c48b47cae55c52c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9db5cb55d57244689c48b47cae55c52c 2024-12-02T21:18:09,107 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9db5cb55d57244689c48b47cae55c52c, entries=150, sequenceid=187, filesize=30.4 K 2024-12-02T21:18:09,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/386c5469af71438bad57cb72b16322d3 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/386c5469af71438bad57cb72b16322d3 2024-12-02T21:18:09,110 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/386c5469af71438bad57cb72b16322d3, entries=150, sequenceid=187, filesize=11.9 K 2024-12-02T21:18:09,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/9729612020fe4e65b06290c687ecf581 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9729612020fe4e65b06290c687ecf581 2024-12-02T21:18:09,112 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9729612020fe4e65b06290c687ecf581, entries=150, sequenceid=187, filesize=11.9 K 2024-12-02T21:18:09,113 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 6cc723b24523dcb8953ed05d209a77e3 in 458ms, sequenceid=187, compaction requested=true 2024-12-02T21:18:09,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:09,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-02T21:18:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-02T21:18:09,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-02T21:18:09,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 613 msec 2024-12-02T21:18:09,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 618 msec 2024-12-02T21:18:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T21:18:09,128 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:09,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:18:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-02T21:18:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T21:18:09,130 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:18:09,131 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:18:09,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:18:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:09,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T21:18:09,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:09,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:09,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:09,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:09,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:09,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:09,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202baa288abccb6490db34a59245b47b9b0_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174288833/Put/seqid=0 2024-12-02T21:18:09,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742372_1548 (size=14794) 2024-12-02T21:18:09,165 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:09,168 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202baa288abccb6490db34a59245b47b9b0_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202baa288abccb6490db34a59245b47b9b0_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:09,168 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/9cc97f29afe94723aa8b13d4b9187144, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:09,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/9cc97f29afe94723aa8b13d4b9187144 is 175, key is test_row_0/A:col10/1733174288833/Put/seqid=0 2024-12-02T21:18:09,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174349166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742373_1549 (size=39749) 2024-12-02T21:18:09,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174349170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174349170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174349171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174349171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T21:18:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T21:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174349278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174349278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174349278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174349278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174349278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T21:18:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T21:18:09,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174349488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174349488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174349488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174349489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174349489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,573 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/9cc97f29afe94723aa8b13d4b9187144 2024-12-02T21:18:09,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/7834543e9ac34e118b5e7111ea9fbb2c is 50, key is test_row_0/B:col10/1733174288833/Put/seqid=0 2024-12-02T21:18:09,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742374_1550 (size=12151) 2024-12-02T21:18:09,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T21:18:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T21:18:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T21:18:09,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174349799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174349799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174349799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174349799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174349799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:09,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T21:18:09,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:09,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:09,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:09,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/7834543e9ac34e118b5e7111ea9fbb2c 2024-12-02T21:18:09,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/6ace4064fc804f7c8ed5f80329eebfb7 is 50, key is test_row_0/C:col10/1733174288833/Put/seqid=0 2024-12-02T21:18:09,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742375_1551 (size=12151) 2024-12-02T21:18:09,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/6ace4064fc804f7c8ed5f80329eebfb7 2024-12-02T21:18:09,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/9cc97f29afe94723aa8b13d4b9187144 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9cc97f29afe94723aa8b13d4b9187144 2024-12-02T21:18:09,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9cc97f29afe94723aa8b13d4b9187144, entries=200, sequenceid=205, filesize=38.8 K 2024-12-02T21:18:09,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/7834543e9ac34e118b5e7111ea9fbb2c as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/7834543e9ac34e118b5e7111ea9fbb2c 2024-12-02T21:18:09,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/7834543e9ac34e118b5e7111ea9fbb2c, entries=150, sequenceid=205, filesize=11.9 K 2024-12-02T21:18:10,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/6ace4064fc804f7c8ed5f80329eebfb7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6ace4064fc804f7c8ed5f80329eebfb7 2024-12-02T21:18:10,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6ace4064fc804f7c8ed5f80329eebfb7, entries=150, sequenceid=205, filesize=11.9 K 2024-12-02T21:18:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6cc723b24523dcb8953ed05d209a77e3 in 851ms, sequenceid=205, compaction requested=true 2024-12-02T21:18:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:18:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:10,004 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-02T21:18:10,004 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-02T21:18:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:18:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:18:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:18:10,006 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 226285 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-02T21:18:10,006 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/A is initiating minor compaction (all files) 2024-12-02T21:18:10,006 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/A in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:10,006 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84963 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-02T21:18:10,006 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/4b29a27b71f446599eec33c5175f4b19, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/556c7667c4824c7592aec82f6668ece3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/fc1f5bec9f29498495259d93728b15fd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6af085a7dcdf488f859f4a7e2d46afe7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/858f8b9686a84e3f9ac6dbd3e0e88821, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9db5cb55d57244689c48b47cae55c52c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9cc97f29afe94723aa8b13d4b9187144] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=221.0 K 2024-12-02T21:18:10,006 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/B is initiating minor compaction (all files) 2024-12-02T21:18:10,006 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:10,006 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/B in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:10,006 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/4b29a27b71f446599eec33c5175f4b19, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/556c7667c4824c7592aec82f6668ece3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/fc1f5bec9f29498495259d93728b15fd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6af085a7dcdf488f859f4a7e2d46afe7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/858f8b9686a84e3f9ac6dbd3e0e88821, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9db5cb55d57244689c48b47cae55c52c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9cc97f29afe94723aa8b13d4b9187144] 2024-12-02T21:18:10,006 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/f9e77bb21a33458194e1a5edd183cd10, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8feeb01d9c2e4ee893cf38f3c61df395, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/72a08964c05045f3bd0b453cd79b43f5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b804a4407cf747499718f2fa68c1ee8c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b611c3ab2246444eaff0cd0116dfc079, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/386c5469af71438bad57cb72b16322d3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/7834543e9ac34e118b5e7111ea9fbb2c] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=83.0 K 2024-12-02T21:18:10,006 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b29a27b71f446599eec33c5175f4b19, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733174280172 2024-12-02T21:18:10,006 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e77bb21a33458194e1a5edd183cd10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733174280172 2024-12-02T21:18:10,007 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 556c7667c4824c7592aec82f6668ece3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733174280507 2024-12-02T21:18:10,007 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 8feeb01d9c2e4ee893cf38f3c61df395, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733174280507 2024-12-02T21:18:10,007 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc1f5bec9f29498495259d93728b15fd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733174281150 2024-12-02T21:18:10,007 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a08964c05045f3bd0b453cd79b43f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733174281150 2024-12-02T21:18:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,007 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6af085a7dcdf488f859f4a7e2d46afe7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733174283318 2024-12-02T21:18:10,007 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b804a4407cf747499718f2fa68c1ee8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733174283318 2024-12-02T21:18:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,008 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 858f8b9686a84e3f9ac6dbd3e0e88821, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733174285474 2024-12-02T21:18:10,008 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting b611c3ab2246444eaff0cd0116dfc079, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733174285474 2024-12-02T21:18:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,008 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 386c5469af71438bad57cb72b16322d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733174287667 2024-12-02T21:18:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,008 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 7834543e9ac34e118b5e7111ea9fbb2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733174288832 2024-12-02T21:18:10,008 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db5cb55d57244689c48b47cae55c52c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733174287667 2024-12-02T21:18:10,009 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc97f29afe94723aa8b13d4b9187144, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733174288832 2024-12-02T21:18:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,021 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#B#compaction#464 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,022 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/04767a49fbe146fcb1d7e6993ccb2a8b is 50, key is test_row_0/B:col10/1733174288833/Put/seqid=0 2024-12-02T21:18:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,029 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:10,031 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202b13a3925018c4db2b388157f426afd1a_6cc723b24523dcb8953ed05d209a77e3 store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,035 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202b13a3925018c4db2b388157f426afd1a_6cc723b24523dcb8953ed05d209a77e3, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:10,035 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b13a3925018c4db2b388157f426afd1a_6cc723b24523dcb8953ed05d209a77e3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742376_1552 (size=12595) 2024-12-02T21:18:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,045 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/04767a49fbe146fcb1d7e6993ccb2a8b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/04767a49fbe146fcb1d7e6993ccb2a8b 2024-12-02T21:18:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T21:18:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,046 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T21:18:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:10,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,049 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/B of 6cc723b24523dcb8953ed05d209a77e3 into 04767a49fbe146fcb1d7e6993ccb2a8b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:10,049 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:10,049 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/B, priority=9, startTime=1733174290004; duration=0sec 2024-12-02T21:18:10,050 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:18:10,050 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:B 2024-12-02T21:18:10,050 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-02T21:18:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,051 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84963 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-02T21:18:10,051 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/C is initiating minor compaction (all files) 2024-12-02T21:18:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,051 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/C in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:10,051 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/00a76cd790a949f4935185cf93577ae4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/58e1b7cc3c92475a8863c8ba751cf3cf, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/41db991a8e674d65a56638652e6d5d3a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/85bd4ea8b4724533883d7341c4388efe, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac9d39388569473ab60ab3886b2f0e7f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9729612020fe4e65b06290c687ecf581, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6ace4064fc804f7c8ed5f80329eebfb7] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=83.0 K 2024-12-02T21:18:10,051 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a76cd790a949f4935185cf93577ae4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733174280172 2024-12-02T21:18:10,052 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e1b7cc3c92475a8863c8ba751cf3cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733174280507 2024-12-02T21:18:10,052 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 41db991a8e674d65a56638652e6d5d3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733174281150 2024-12-02T21:18:10,052 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 85bd4ea8b4724533883d7341c4388efe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733174283318 2024-12-02T21:18:10,052 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ac9d39388569473ab60ab3886b2f0e7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733174285474 2024-12-02T21:18:10,052 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 9729612020fe4e65b06290c687ecf581, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733174287667 2024-12-02T21:18:10,053 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ace4064fc804f7c8ed5f80329eebfb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733174288832 2024-12-02T21:18:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742377_1553 (size=4469) 2024-12-02T21:18:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,073 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#C#compaction#466 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:10,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025b1f9824fb7049a1a515be9e31f517dc_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174289169/Put/seqid=0 2024-12-02T21:18:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,073 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/6100c758a5e1428799d8769881c8bd99 is 50, key is test_row_0/C:col10/1733174288833/Put/seqid=0 2024-12-02T21:18:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742379_1555 (size=12595) 2024-12-02T21:18:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742378_1554 (size=9814) 2024-12-02T21:18:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,112 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/6100c758a5e1428799d8769881c8bd99 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6100c758a5e1428799d8769881c8bd99 2024-12-02T21:18:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,113 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025b1f9824fb7049a1a515be9e31f517dc_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025b1f9824fb7049a1a515be9e31f517dc_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/678197540cde459085a20e0a6f06c676, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/678197540cde459085a20e0a6f06c676 is 175, key is test_row_0/A:col10/1733174289169/Put/seqid=0 2024-12-02T21:18:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,117 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/C of 6cc723b24523dcb8953ed05d209a77e3 into 6100c758a5e1428799d8769881c8bd99(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:10,117 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:10,117 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/C, priority=9, startTime=1733174290004; duration=0sec 2024-12-02T21:18:10,117 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:10,117 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:C 2024-12-02T21:18:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742380_1556 (size=22461) 2024-12-02T21:18:10,122 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/678197540cde459085a20e0a6f06c676 2024-12-02T21:18:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/0a945037ac734afe9d39d045706490de is 50, key is test_row_0/B:col10/1733174289169/Put/seqid=0 2024-12-02T21:18:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742381_1557 (size=9757) 2024-12-02T21:18:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,136 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/0a945037ac734afe9d39d045706490de 2024-12-02T21:18:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ac2363db2fe74a3399b92d41740d068a is 50, key is test_row_0/C:col10/1733174289169/Put/seqid=0 2024-12-02T21:18:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742382_1558 (size=9757) 2024-12-02T21:18:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T21:18:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:10,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:10,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174350354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174350355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174350356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174350357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174350360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,463 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:10,463 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/cfed9fcf2d2d49fe97020f43dfe0acb9 is 175, key is test_row_0/A:col10/1733174288833/Put/seqid=0 2024-12-02T21:18:10,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742383_1559 (size=31549) 2024-12-02T21:18:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174350469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174350469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174350469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174350470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174350470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,546 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ac2363db2fe74a3399b92d41740d068a 2024-12-02T21:18:10,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/678197540cde459085a20e0a6f06c676 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/678197540cde459085a20e0a6f06c676 2024-12-02T21:18:10,551 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/678197540cde459085a20e0a6f06c676, entries=100, sequenceid=219, filesize=21.9 K 2024-12-02T21:18:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/0a945037ac734afe9d39d045706490de as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/0a945037ac734afe9d39d045706490de 2024-12-02T21:18:10,554 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/0a945037ac734afe9d39d045706490de, entries=100, sequenceid=219, filesize=9.5 K 2024-12-02T21:18:10,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ac2363db2fe74a3399b92d41740d068a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac2363db2fe74a3399b92d41740d068a 2024-12-02T21:18:10,557 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac2363db2fe74a3399b92d41740d068a, entries=100, sequenceid=219, filesize=9.5 K 2024-12-02T21:18:10,558 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6cc723b24523dcb8953ed05d209a77e3 in 512ms, sequenceid=219, compaction requested=false 2024-12-02T21:18:10,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:10,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:10,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-02T21:18:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-02T21:18:10,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-02T21:18:10,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4280 sec 2024-12-02T21:18:10,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.4340 sec 2024-12-02T21:18:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:10,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T21:18:10,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:10,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:10,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:10,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:10,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:10,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:10,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202778edcdc0b844f2b804adf973bcb701d_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174290353/Put/seqid=0 2024-12-02T21:18:10,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742384_1560 (size=12304) 2024-12-02T21:18:10,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174350689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174350690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174350691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,696 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174350693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174350694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,699 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202778edcdc0b844f2b804adf973bcb701d_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202778edcdc0b844f2b804adf973bcb701d_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:10,700 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/71834ef7954f4f04b060171955343ff2, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:10,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/71834ef7954f4f04b060171955343ff2 is 175, key is test_row_0/A:col10/1733174290353/Put/seqid=0 2024-12-02T21:18:10,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742385_1561 (size=31105) 2024-12-02T21:18:10,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174350799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174350799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174350799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174350808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174350808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:10,870 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/cfed9fcf2d2d49fe97020f43dfe0acb9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/cfed9fcf2d2d49fe97020f43dfe0acb9 2024-12-02T21:18:10,873 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/A of 6cc723b24523dcb8953ed05d209a77e3 into cfed9fcf2d2d49fe97020f43dfe0acb9(size=30.8 K), total size for store is 52.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:10,873 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:10,873 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/A, priority=9, startTime=1733174290004; duration=0sec 2024-12-02T21:18:10,873 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:10,873 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:A 2024-12-02T21:18:11,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174351009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174351009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174351009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174351018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174351018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,103 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/71834ef7954f4f04b060171955343ff2 2024-12-02T21:18:11,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/ad49f684d597438f976bb3f1a354954a is 50, key is test_row_0/B:col10/1733174290353/Put/seqid=0 2024-12-02T21:18:11,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742386_1562 (size=12151) 2024-12-02T21:18:11,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/ad49f684d597438f976bb3f1a354954a 2024-12-02T21:18:11,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ff419eb67f8f4f8f86dd2d0f99564530 is 50, key is test_row_0/C:col10/1733174290353/Put/seqid=0 2024-12-02T21:18:11,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742387_1563 (size=12151) 2024-12-02T21:18:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T21:18:11,279 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:11,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:18:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-02T21:18:11,281 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:18:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T21:18:11,281 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:18:11,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:18:11,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174351319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174351319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174351319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174351329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174351329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T21:18:11,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T21:18:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:11,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:11,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:11,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ff419eb67f8f4f8f86dd2d0f99564530 2024-12-02T21:18:11,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/71834ef7954f4f04b060171955343ff2 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71834ef7954f4f04b060171955343ff2 2024-12-02T21:18:11,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71834ef7954f4f04b060171955343ff2, entries=150, sequenceid=241, filesize=30.4 K 2024-12-02T21:18:11,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/ad49f684d597438f976bb3f1a354954a as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/ad49f684d597438f976bb3f1a354954a 2024-12-02T21:18:11,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/ad49f684d597438f976bb3f1a354954a, entries=150, sequenceid=241, filesize=11.9 K 2024-12-02T21:18:11,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/ff419eb67f8f4f8f86dd2d0f99564530 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ff419eb67f8f4f8f86dd2d0f99564530 2024-12-02T21:18:11,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ff419eb67f8f4f8f86dd2d0f99564530, entries=150, sequenceid=241, filesize=11.9 K 2024-12-02T21:18:11,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6cc723b24523dcb8953ed05d209a77e3 in 857ms, sequenceid=241, compaction requested=true 2024-12-02T21:18:11,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:11,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:18:11,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:11,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:18:11,538 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:18:11,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:11,538 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:18:11,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:18:11,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:18:11,539 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:18:11,539 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:18:11,539 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/B is initiating minor compaction (all files) 2024-12-02T21:18:11,539 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/A is initiating minor compaction (all files) 2024-12-02T21:18:11,539 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/B in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:11,539 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/04767a49fbe146fcb1d7e6993ccb2a8b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/0a945037ac734afe9d39d045706490de, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/ad49f684d597438f976bb3f1a354954a] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=33.7 K 2024-12-02T21:18:11,539 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/A in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:11,539 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/cfed9fcf2d2d49fe97020f43dfe0acb9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/678197540cde459085a20e0a6f06c676, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71834ef7954f4f04b060171955343ff2] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=83.1 K 2024-12-02T21:18:11,539 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:11,539 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/cfed9fcf2d2d49fe97020f43dfe0acb9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/678197540cde459085a20e0a6f06c676, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71834ef7954f4f04b060171955343ff2] 2024-12-02T21:18:11,540 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 04767a49fbe146fcb1d7e6993ccb2a8b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733174288832 2024-12-02T21:18:11,540 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfed9fcf2d2d49fe97020f43dfe0acb9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733174288832 2024-12-02T21:18:11,540 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678197540cde459085a20e0a6f06c676, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733174289169 2024-12-02T21:18:11,540 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71834ef7954f4f04b060171955343ff2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733174290353 2024-12-02T21:18:11,540 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a945037ac734afe9d39d045706490de, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733174289169 2024-12-02T21:18:11,541 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ad49f684d597438f976bb3f1a354954a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733174290353 2024-12-02T21:18:11,545 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:11,547 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:11,547 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/a230b8f2e931483e823d05d93afce977 is 50, key is test_row_0/B:col10/1733174290353/Put/seqid=0 2024-12-02T21:18:11,561 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120265b8e7431cac45e4aecb414aa8e8eb80_6cc723b24523dcb8953ed05d209a77e3 store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:11,563 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120265b8e7431cac45e4aecb414aa8e8eb80_6cc723b24523dcb8953ed05d209a77e3, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:11,563 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120265b8e7431cac45e4aecb414aa8e8eb80_6cc723b24523dcb8953ed05d209a77e3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:11,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742388_1564 (size=12697) 2024-12-02T21:18:11,577 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/a230b8f2e931483e823d05d93afce977 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a230b8f2e931483e823d05d93afce977 2024-12-02T21:18:11,583 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/B of 6cc723b24523dcb8953ed05d209a77e3 into a230b8f2e931483e823d05d93afce977(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:11,583 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:11,583 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/B, priority=13, startTime=1733174291538; duration=0sec 2024-12-02T21:18:11,583 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:18:11,583 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:B 2024-12-02T21:18:11,583 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:18:11,585 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:18:11,585 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/C is initiating minor compaction (all files) 2024-12-02T21:18:11,585 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/C in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:11,585 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6100c758a5e1428799d8769881c8bd99, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac2363db2fe74a3399b92d41740d068a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ff419eb67f8f4f8f86dd2d0f99564530] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=33.7 K 2024-12-02T21:18:11,586 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6100c758a5e1428799d8769881c8bd99, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733174288832 2024-12-02T21:18:11,586 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ac2363db2fe74a3399b92d41740d068a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733174289169 2024-12-02T21:18:11,586 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting ff419eb67f8f4f8f86dd2d0f99564530, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733174290353 2024-12-02T21:18:11,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T21:18:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:11,587 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T21:18:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:11,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742389_1565 (size=4469) 2024-12-02T21:18:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T21:18:11,599 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#C#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:11,600 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/5c601d41076a40e4b4352e44975c009d is 50, key is test_row_0/C:col10/1733174290353/Put/seqid=0 2024-12-02T21:18:11,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202df8e362089354819b6c1a4e78dd62320_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174290693/Put/seqid=0 2024-12-02T21:18:11,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742390_1566 (size=12697) 2024-12-02T21:18:11,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742391_1567 (size=12304) 2024-12-02T21:18:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:11,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:11,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174351867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174351867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174291866, max timestamp=1733174291872Num uniques 859; 2024-12-02T21:18:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:18:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174291866, max timestamp=1733174291872Num uniques 859; 2024-12-02T21:18:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733174291866, max timestamp=1733174291872Num uniques 859; 2024-12-02T21:18:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:18:11,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87c3fdb6c570:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T21:18:11,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174351898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174351898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174351899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T21:18:11,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174351978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174351978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:11,999 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#A#compaction#473 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:11,999 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/062ab8bcc3c8455e99da8161594c2888 is 175, key is test_row_0/A:col10/1733174290353/Put/seqid=0 2024-12-02T21:18:12,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742392_1568 (size=31651) 2024-12-02T21:18:12,008 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/062ab8bcc3c8455e99da8161594c2888 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/062ab8bcc3c8455e99da8161594c2888 2024-12-02T21:18:12,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174352009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174352009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174352010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,015 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/5c601d41076a40e4b4352e44975c009d as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5c601d41076a40e4b4352e44975c009d 2024-12-02T21:18:12,017 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202df8e362089354819b6c1a4e78dd62320_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202df8e362089354819b6c1a4e78dd62320_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/6c7f5753de2e4b4c94643d5023a542de, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/6c7f5753de2e4b4c94643d5023a542de is 175, key is test_row_0/A:col10/1733174290693/Put/seqid=0 2024-12-02T21:18:12,025 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/C of 6cc723b24523dcb8953ed05d209a77e3 into 5c601d41076a40e4b4352e44975c009d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:12,025 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/A of 6cc723b24523dcb8953ed05d209a77e3 into 062ab8bcc3c8455e99da8161594c2888(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:12,025 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:12,025 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:12,025 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/A, priority=13, startTime=1733174291538; duration=0sec 2024-12-02T21:18:12,025 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/C, priority=13, startTime=1733174291538; duration=0sec 2024-12-02T21:18:12,025 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:12,025 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:C 2024-12-02T21:18:12,026 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:12,026 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:A 2024-12-02T21:18:12,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742393_1569 (size=31105) 2024-12-02T21:18:12,179 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T21:18:12,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174352189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174352189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174352221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174352221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174352221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T21:18:12,430 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/6c7f5753de2e4b4c94643d5023a542de 2024-12-02T21:18:12,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/391e427995c54048ba63f6016b8289c8 is 50, key is test_row_0/B:col10/1733174290693/Put/seqid=0 2024-12-02T21:18:12,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742394_1570 (size=12151) 2024-12-02T21:18:12,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174352498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174352498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174352529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174352529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174352529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:12,840 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/391e427995c54048ba63f6016b8289c8 2024-12-02T21:18:12,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/eae87c35a28f4555a2b34f5b23e6781e is 50, key is test_row_0/C:col10/1733174290693/Put/seqid=0 2024-12-02T21:18:12,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742395_1571 (size=12151) 2024-12-02T21:18:12,851 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/eae87c35a28f4555a2b34f5b23e6781e 2024-12-02T21:18:12,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/6c7f5753de2e4b4c94643d5023a542de as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6c7f5753de2e4b4c94643d5023a542de 2024-12-02T21:18:12,859 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6c7f5753de2e4b4c94643d5023a542de, entries=150, sequenceid=255, filesize=30.4 K 2024-12-02T21:18:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/391e427995c54048ba63f6016b8289c8 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/391e427995c54048ba63f6016b8289c8 2024-12-02T21:18:12,864 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/391e427995c54048ba63f6016b8289c8, entries=150, sequenceid=255, filesize=11.9 K 2024-12-02T21:18:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/eae87c35a28f4555a2b34f5b23e6781e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eae87c35a28f4555a2b34f5b23e6781e 2024-12-02T21:18:12,867 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eae87c35a28f4555a2b34f5b23e6781e, entries=150, sequenceid=255, filesize=11.9 K 2024-12-02T21:18:12,868 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 6cc723b24523dcb8953ed05d209a77e3 in 1281ms, sequenceid=255, compaction requested=false 2024-12-02T21:18:12,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:12,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:12,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-02T21:18:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-02T21:18:12,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-02T21:18:12,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5890 sec 2024-12-02T21:18:12,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.5930 sec 2024-12-02T21:18:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:13,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-02T21:18:13,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:13,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:18:13,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:13,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:18:13,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:13,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T21:18:13,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120274494a9ea5ae465b8e7fe9eb0df116bd_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174291896/Put/seqid=0 2024-12-02T21:18:13,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742396_1572 (size=14994) 2024-12-02T21:18:13,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174353031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174353032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174353039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174353039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174353039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174353139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174353139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174353348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174353349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T21:18:13,428 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:13,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T21:18:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-02T21:18:13,430 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T21:18:13,431 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T21:18:13,432 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T21:18:13,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T21:18:13,434 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120274494a9ea5ae465b8e7fe9eb0df116bd_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120274494a9ea5ae465b8e7fe9eb0df116bd_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:13,435 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/c7d765959ad74d678603135dde72a937, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:13,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/c7d765959ad74d678603135dde72a937 is 175, key is test_row_0/A:col10/1733174291896/Put/seqid=0 2024-12-02T21:18:13,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742397_1573 (size=39949) 2024-12-02T21:18:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T21:18:13,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T21:18:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:13,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:13,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:13,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:13,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:13,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174353659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174353659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:13,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T21:18:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:13,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T21:18:13,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/c7d765959ad74d678603135dde72a937 2024-12-02T21:18:13,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/6b3c95923e3940f08ceeeabdb10c17a8 is 50, key is test_row_0/B:col10/1733174291896/Put/seqid=0 2024-12-02T21:18:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742398_1574 (size=12301) 2024-12-02T21:18:13,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T21:18:13,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:13,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:13,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:13,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:13,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T21:18:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58586 deadline: 1733174354049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:14,050 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_1' in 6cc723b24523dcb8953ed05d209a77e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:18:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T21:18:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58548 deadline: 1733174354058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58598 deadline: 1733174354058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:14,060 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 6cc723b24523dcb8953ed05d209a77e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,060 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 87c3fdb6c570,35563,1733174113859 for 'test_row_2' in 6cc723b24523dcb8953ed05d209a77e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T21:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:14,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58572 deadline: 1733174354168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:18:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35563 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58554 deadline: 1733174354168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 , the old value is region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T21:18:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., hostname=87c3fdb6c570,35563,1733174113859, seqNum=5 because the exception is null or not the one we care about 2024-12-02T21:18:14,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T21:18:14,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:14,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/6b3c95923e3940f08ceeeabdb10c17a8 2024-12-02T21:18:14,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/eb750fd1e1f7479690f7c759afc0539b is 50, key is test_row_0/C:col10/1733174291896/Put/seqid=0 2024-12-02T21:18:14,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742399_1575 (size=12301) 2024-12-02T21:18:14,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T21:18:14,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:14,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T21:18:14,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:14,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T21:18:14,604 INFO [Thread-2262 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2262. 2024-12-02T21:18:14,604 DEBUG [Thread-2262 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:14,604 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,604 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:14,605 INFO [Thread-2260 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2260. 2024-12-02T21:18:14,605 DEBUG [Thread-2260 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:14,605 DEBUG [Thread-2260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,605 DEBUG [Thread-2260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:14,608 INFO [Thread-2258 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2258. 2024-12-02T21:18:14,608 DEBUG [Thread-2258 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:14,608 DEBUG [Thread-2258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,608 DEBUG [Thread-2258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:14,608 INFO [Thread-2264 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2264. 2024-12-02T21:18:14,608 DEBUG [Thread-2264 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:14,608 DEBUG [Thread-2264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,608 DEBUG [Thread-2264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:14,609 INFO [Thread-2256 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2256. 2024-12-02T21:18:14,609 DEBUG [Thread-2256 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:14,609 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,609 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:14,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:14,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T21:18:14,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:14,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:18:14,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/eb750fd1e1f7479690f7c759afc0539b 2024-12-02T21:18:14,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/c7d765959ad74d678603135dde72a937 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/c7d765959ad74d678603135dde72a937 2024-12-02T21:18:14,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/c7d765959ad74d678603135dde72a937, entries=200, sequenceid=293, filesize=39.0 K 2024-12-02T21:18:14,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/6b3c95923e3940f08ceeeabdb10c17a8 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/6b3c95923e3940f08ceeeabdb10c17a8 2024-12-02T21:18:14,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/6b3c95923e3940f08ceeeabdb10c17a8, entries=150, sequenceid=293, filesize=12.0 K 2024-12-02T21:18:14,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/eb750fd1e1f7479690f7c759afc0539b as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eb750fd1e1f7479690f7c759afc0539b 2024-12-02T21:18:14,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eb750fd1e1f7479690f7c759afc0539b, entries=150, sequenceid=293, filesize=12.0 K 2024-12-02T21:18:14,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 6cc723b24523dcb8953ed05d209a77e3 in 1660ms, sequenceid=293, compaction requested=true 2024-12-02T21:18:14,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:14,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T21:18:14,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:14,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T21:18:14,673 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:18:14,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:14,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6cc723b24523dcb8953ed05d209a77e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T21:18:14,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:18:14,673 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:18:14,673 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:18:14,673 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:18:14,673 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/A is initiating minor compaction (all files) 2024-12-02T21:18:14,673 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/B is initiating minor compaction (all files) 2024-12-02T21:18:14,673 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/B in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,673 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/A in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,673 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a230b8f2e931483e823d05d93afce977, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/391e427995c54048ba63f6016b8289c8, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/6b3c95923e3940f08ceeeabdb10c17a8] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=36.3 K 2024-12-02T21:18:14,673 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/062ab8bcc3c8455e99da8161594c2888, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6c7f5753de2e4b4c94643d5023a542de, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/c7d765959ad74d678603135dde72a937] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=100.3 K 2024-12-02T21:18:14,674 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,674 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. files: [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/062ab8bcc3c8455e99da8161594c2888, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6c7f5753de2e4b4c94643d5023a542de, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/c7d765959ad74d678603135dde72a937] 2024-12-02T21:18:14,674 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting a230b8f2e931483e823d05d93afce977, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733174290353 2024-12-02T21:18:14,674 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 391e427995c54048ba63f6016b8289c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733174290682 2024-12-02T21:18:14,674 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 062ab8bcc3c8455e99da8161594c2888, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733174290353 2024-12-02T21:18:14,674 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b3c95923e3940f08ceeeabdb10c17a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733174291896 2024-12-02T21:18:14,674 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7f5753de2e4b4c94643d5023a542de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733174290682 2024-12-02T21:18:14,674 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d765959ad74d678603135dde72a937, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733174291896 2024-12-02T21:18:14,678 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:14,679 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:14,679 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/392cd8835d634f94b694d52a8b909f40 is 50, key is test_row_0/B:col10/1733174291896/Put/seqid=0 2024-12-02T21:18:14,679 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202f446b2e05b1b4bbbbf2c872c703112a1_6cc723b24523dcb8953ed05d209a77e3 store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:14,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742400_1576 (size=12949) 2024-12-02T21:18:14,685 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202f446b2e05b1b4bbbbf2c872c703112a1_6cc723b24523dcb8953ed05d209a77e3, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:14,685 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f446b2e05b1b4bbbbf2c872c703112a1_6cc723b24523dcb8953ed05d209a77e3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:14,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742401_1577 (size=4469) 2024-12-02T21:18:14,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35563 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T21:18:14,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:14,807 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T21:18:14,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:14,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:14,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:14,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:14,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:14,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:14,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120257070e1e3f1b421bb3da984eee98794e_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174293018/Put/seqid=0 2024-12-02T21:18:14,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742402_1578 (size=12454) 2024-12-02T21:18:15,085 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/392cd8835d634f94b694d52a8b909f40 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/392cd8835d634f94b694d52a8b909f40 2024-12-02T21:18:15,088 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/B of 6cc723b24523dcb8953ed05d209a77e3 into 392cd8835d634f94b694d52a8b909f40(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:15,088 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:15,088 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/B, priority=13, startTime=1733174294673; duration=0sec 2024-12-02T21:18:15,088 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T21:18:15,088 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:B 2024-12-02T21:18:15,088 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T21:18:15,089 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T21:18:15,089 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1541): 6cc723b24523dcb8953ed05d209a77e3/C is initiating minor compaction (all files) 2024-12-02T21:18:15,089 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6cc723b24523dcb8953ed05d209a77e3/C in TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:15,089 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5c601d41076a40e4b4352e44975c009d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eae87c35a28f4555a2b34f5b23e6781e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eb750fd1e1f7479690f7c759afc0539b] into tmpdir=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp, totalSize=36.3 K 2024-12-02T21:18:15,089 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c601d41076a40e4b4352e44975c009d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733174290353 2024-12-02T21:18:15,090 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting eae87c35a28f4555a2b34f5b23e6781e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733174290682 2024-12-02T21:18:15,090 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] compactions.Compactor(225): Compacting eb750fd1e1f7479690f7c759afc0539b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733174291896 2024-12-02T21:18:15,094 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#C#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:15,094 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/25bde5e7491a4ffc82877d35d82e0dab is 50, key is test_row_0/C:col10/1733174291896/Put/seqid=0 2024-12-02T21:18:15,094 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6cc723b24523dcb8953ed05d209a77e3#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T21:18:15,095 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/66bd1e31d43046b2abc29c3dde7da6a1 is 175, key is test_row_0/A:col10/1733174291896/Put/seqid=0 2024-12-02T21:18:15,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742403_1579 (size=12949) 2024-12-02T21:18:15,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742404_1580 (size=31903) 2024-12-02T21:18:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35563 {}] regionserver.HRegion(8855): Flush requested on 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:15,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. as already flushing 2024-12-02T21:18:15,180 INFO [Thread-2253 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2253. 2024-12-02T21:18:15,180 INFO [Thread-2245 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2245. 2024-12-02T21:18:15,180 DEBUG [Thread-2253 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:15,180 DEBUG [Thread-2253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:15,180 DEBUG [Thread-2245 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:15,180 DEBUG [Thread-2245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:15,181 DEBUG [Thread-2253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:15,181 DEBUG [Thread-2245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:15,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:15,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:15,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:15,219 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120257070e1e3f1b421bb3da984eee98794e_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120257070e1e3f1b421bb3da984eee98794e_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/1bc62ed450b9472b9cdccfd175900b75, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/1bc62ed450b9472b9cdccfd175900b75 is 175, key is test_row_0/A:col10/1733174293018/Put/seqid=0 2024-12-02T21:18:15,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742405_1581 (size=31255) 2024-12-02T21:18:15,500 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/25bde5e7491a4ffc82877d35d82e0dab as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/25bde5e7491a4ffc82877d35d82e0dab 2024-12-02T21:18:15,500 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/66bd1e31d43046b2abc29c3dde7da6a1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/66bd1e31d43046b2abc29c3dde7da6a1 2024-12-02T21:18:15,503 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/C of 6cc723b24523dcb8953ed05d209a77e3 into 25bde5e7491a4ffc82877d35d82e0dab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:15,503 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:15,503 INFO [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/C, priority=13, startTime=1733174294673; duration=0sec 2024-12-02T21:18:15,503 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:15,503 DEBUG [RS:0;87c3fdb6c570:35563-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:C 2024-12-02T21:18:15,504 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6cc723b24523dcb8953ed05d209a77e3/A of 6cc723b24523dcb8953ed05d209a77e3 into 66bd1e31d43046b2abc29c3dde7da6a1(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T21:18:15,504 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:15,504 INFO [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., storeName=6cc723b24523dcb8953ed05d209a77e3/A, priority=13, startTime=1733174294672; duration=0sec 2024-12-02T21:18:15,504 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T21:18:15,504 DEBUG [RS:0;87c3fdb6c570:35563-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6cc723b24523dcb8953ed05d209a77e3:A 2024-12-02T21:18:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T21:18:15,623 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/1bc62ed450b9472b9cdccfd175900b75 2024-12-02T21:18:15,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/a7d95725fe224bbda3f497550b2465b2 is 50, key is test_row_0/B:col10/1733174293018/Put/seqid=0 2024-12-02T21:18:15,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742406_1582 (size=12301) 2024-12-02T21:18:16,032 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/a7d95725fe224bbda3f497550b2465b2 2024-12-02T21:18:16,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/3c05ccd45a4244e287ca7c485852aee1 is 50, key is test_row_0/C:col10/1733174293018/Put/seqid=0 2024-12-02T21:18:16,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742407_1583 (size=12301) 2024-12-02T21:18:16,060 INFO [Thread-2247 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2247. 2024-12-02T21:18:16,060 DEBUG [Thread-2247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:16,060 DEBUG [Thread-2247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:16,060 DEBUG [Thread-2247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:16,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:16,070 INFO [Thread-2249 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2249. 2024-12-02T21:18:16,070 DEBUG [Thread-2249 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:16,070 DEBUG [Thread-2249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:16,070 DEBUG [Thread-2249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:16,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:16,090 INFO [Thread-2251 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2251. 2024-12-02T21:18:16,090 DEBUG [Thread-2251 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T21:18:16,090 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:16,090 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:16,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:16,441 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/3c05ccd45a4244e287ca7c485852aee1 2024-12-02T21:18:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/1bc62ed450b9472b9cdccfd175900b75 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/1bc62ed450b9472b9cdccfd175900b75 2024-12-02T21:18:16,446 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/1bc62ed450b9472b9cdccfd175900b75, entries=150, sequenceid=304, filesize=30.5 K 2024-12-02T21:18:16,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/a7d95725fe224bbda3f497550b2465b2 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a7d95725fe224bbda3f497550b2465b2 2024-12-02T21:18:16,448 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a7d95725fe224bbda3f497550b2465b2, entries=150, sequenceid=304, filesize=12.0 K 2024-12-02T21:18:16,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/3c05ccd45a4244e287ca7c485852aee1 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/3c05ccd45a4244e287ca7c485852aee1 2024-12-02T21:18:16,451 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/3c05ccd45a4244e287ca7c485852aee1, entries=150, sequenceid=304, filesize=12.0 K 2024-12-02T21:18:16,452 INFO [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=33.54 KB/34350 for 6cc723b24523dcb8953ed05d209a77e3 in 1645ms, sequenceid=304, compaction requested=false 2024-12-02T21:18:16,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 6cc723b24523dcb8953ed05d209a77e3: 2024-12-02T21:18:16,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:16,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-02T21:18:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-02T21:18:16,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-02T21:18:16,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0210 sec 2024-12-02T21:18:16,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 3.0260 sec 2024-12-02T21:18:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T21:18:17,599 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:17,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T21:18:17,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-02T21:18:17,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-02T21:18:17,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-02T21:18:17,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-02T21:18:17,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-12-02T21:18:17,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T21:18:17,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4278 2024-12-02T21:18:17,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4052 2024-12-02T21:18:17,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4062 2024-12-02T21:18:17,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4297 2024-12-02T21:18:17,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4034 2024-12-02T21:18:17,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T21:18:17,600 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:18:17,600 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:18:17,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:17,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:17,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:17,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T21:18:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T21:18:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T21:18:17,603 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174297603"}]},"ts":"1733174297603"} 2024-12-02T21:18:17,604 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T21:18:17,604 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T21:18:17,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T21:18:17,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, UNASSIGN}] 2024-12-02T21:18:17,606 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, UNASSIGN 2024-12-02T21:18:17,606 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=6cc723b24523dcb8953ed05d209a77e3, regionState=CLOSING, regionLocation=87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:17,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, UNASSIGN because future has completed 2024-12-02T21:18:17,608 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:18:17,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859}] 2024-12-02T21:18:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T21:18:17,759 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:17,759 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 6cc723b24523dcb8953ed05d209a77e3, disabling compactions & flushes 2024-12-02T21:18:17,760 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. after waiting 0 ms 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:17,760 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing 6cc723b24523dcb8953ed05d209a77e3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=A 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=B 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6cc723b24523dcb8953ed05d209a77e3, store=C 2024-12-02T21:18:17,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T21:18:17,764 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202120f7533e37e4c6896e3d7c5ffd9857c_6cc723b24523dcb8953ed05d209a77e3 is 50, key is test_row_0/A:col10/1733174295179/Put/seqid=0 2024-12-02T21:18:17,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742408_1584 (size=12454) 2024-12-02T21:18:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T21:18:18,168 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:18:18,171 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202120f7533e37e4c6896e3d7c5ffd9857c_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202120f7533e37e4c6896e3d7c5ffd9857c_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:18,171 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/2afc97cb75de4cabaeb2b1678fcb510e, store: [table=TestAcidGuarantees family=A region=6cc723b24523dcb8953ed05d209a77e3] 2024-12-02T21:18:18,172 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/2afc97cb75de4cabaeb2b1678fcb510e is 175, key is test_row_0/A:col10/1733174295179/Put/seqid=0 2024-12-02T21:18:18,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742409_1585 (size=31255) 2024-12-02T21:18:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T21:18:18,575 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/2afc97cb75de4cabaeb2b1678fcb510e 2024-12-02T21:18:18,580 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/78be44634b5c44c7b8dbd9616dfa3b66 is 50, key is test_row_0/B:col10/1733174295179/Put/seqid=0 2024-12-02T21:18:18,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742410_1586 (size=12301) 2024-12-02T21:18:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T21:18:18,983 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/78be44634b5c44c7b8dbd9616dfa3b66 2024-12-02T21:18:18,987 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/7d783eebdc254512b43429069d39fecd is 50, key is test_row_0/C:col10/1733174295179/Put/seqid=0 2024-12-02T21:18:18,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742411_1587 (size=12301) 2024-12-02T21:18:19,391 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/7d783eebdc254512b43429069d39fecd 2024-12-02T21:18:19,394 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/A/2afc97cb75de4cabaeb2b1678fcb510e as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/2afc97cb75de4cabaeb2b1678fcb510e 2024-12-02T21:18:19,397 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/2afc97cb75de4cabaeb2b1678fcb510e, entries=150, sequenceid=315, filesize=30.5 K 2024-12-02T21:18:19,397 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/B/78be44634b5c44c7b8dbd9616dfa3b66 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/78be44634b5c44c7b8dbd9616dfa3b66 2024-12-02T21:18:19,400 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/78be44634b5c44c7b8dbd9616dfa3b66, entries=150, sequenceid=315, filesize=12.0 K 2024-12-02T21:18:19,400 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/.tmp/C/7d783eebdc254512b43429069d39fecd as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/7d783eebdc254512b43429069d39fecd 2024-12-02T21:18:19,402 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/7d783eebdc254512b43429069d39fecd, entries=150, sequenceid=315, filesize=12.0 K 2024-12-02T21:18:19,403 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6cc723b24523dcb8953ed05d209a77e3 in 1643ms, sequenceid=315, compaction requested=true 2024-12-02T21:18:19,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/0bd638990c2a47db947c19bd9f545437, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/8db1cf9c05874f0eb1ecf0c01d36cbff, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/7cda686370f74322b27e77b8509d8db6, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71a7eed705ac454db6feb8d8435fa632, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/938980e52da648488a3b7f2e2aa7b172, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/4b29a27b71f446599eec33c5175f4b19, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/a3428e8fcb89464782abecf0f28aa936, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/556c7667c4824c7592aec82f6668ece3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/fc1f5bec9f29498495259d93728b15fd, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6af085a7dcdf488f859f4a7e2d46afe7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/858f8b9686a84e3f9ac6dbd3e0e88821, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9db5cb55d57244689c48b47cae55c52c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9cc97f29afe94723aa8b13d4b9187144, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/cfed9fcf2d2d49fe97020f43dfe0acb9, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/678197540cde459085a20e0a6f06c676, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/062ab8bcc3c8455e99da8161594c2888, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71834ef7954f4f04b060171955343ff2, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6c7f5753de2e4b4c94643d5023a542de, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/c7d765959ad74d678603135dde72a937] to archive 2024-12-02T21:18:19,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:18:19,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/0bd638990c2a47db947c19bd9f545437 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/0bd638990c2a47db947c19bd9f545437 2024-12-02T21:18:19,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/8db1cf9c05874f0eb1ecf0c01d36cbff to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/8db1cf9c05874f0eb1ecf0c01d36cbff 2024-12-02T21:18:19,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/7cda686370f74322b27e77b8509d8db6 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/7cda686370f74322b27e77b8509d8db6 2024-12-02T21:18:19,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71a7eed705ac454db6feb8d8435fa632 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71a7eed705ac454db6feb8d8435fa632 2024-12-02T21:18:19,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/938980e52da648488a3b7f2e2aa7b172 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/938980e52da648488a3b7f2e2aa7b172 2024-12-02T21:18:19,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/4b29a27b71f446599eec33c5175f4b19 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/4b29a27b71f446599eec33c5175f4b19 2024-12-02T21:18:19,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/a3428e8fcb89464782abecf0f28aa936 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/a3428e8fcb89464782abecf0f28aa936 2024-12-02T21:18:19,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/556c7667c4824c7592aec82f6668ece3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/556c7667c4824c7592aec82f6668ece3 2024-12-02T21:18:19,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/fc1f5bec9f29498495259d93728b15fd to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/fc1f5bec9f29498495259d93728b15fd 2024-12-02T21:18:19,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6af085a7dcdf488f859f4a7e2d46afe7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6af085a7dcdf488f859f4a7e2d46afe7 2024-12-02T21:18:19,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/858f8b9686a84e3f9ac6dbd3e0e88821 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/858f8b9686a84e3f9ac6dbd3e0e88821 2024-12-02T21:18:19,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9db5cb55d57244689c48b47cae55c52c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9db5cb55d57244689c48b47cae55c52c 2024-12-02T21:18:19,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9cc97f29afe94723aa8b13d4b9187144 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/9cc97f29afe94723aa8b13d4b9187144 2024-12-02T21:18:19,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/cfed9fcf2d2d49fe97020f43dfe0acb9 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/cfed9fcf2d2d49fe97020f43dfe0acb9 2024-12-02T21:18:19,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/678197540cde459085a20e0a6f06c676 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/678197540cde459085a20e0a6f06c676 2024-12-02T21:18:19,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/062ab8bcc3c8455e99da8161594c2888 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/062ab8bcc3c8455e99da8161594c2888 2024-12-02T21:18:19,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71834ef7954f4f04b060171955343ff2 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/71834ef7954f4f04b060171955343ff2 2024-12-02T21:18:19,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6c7f5753de2e4b4c94643d5023a542de to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/6c7f5753de2e4b4c94643d5023a542de 2024-12-02T21:18:19,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/c7d765959ad74d678603135dde72a937 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/c7d765959ad74d678603135dde72a937 2024-12-02T21:18:19,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/fa97b4586c8b4cf983ff890bc8a4af95, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a989d6e06ca546e380330de726a1fa7a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/5530a4537e464b1299f9ed41d122c35b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8e1d32a256dd4c2cb38ee69ab9f2f636, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/cfb02b910b8a4e498ec03788d3708a52, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/f9e77bb21a33458194e1a5edd183cd10, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/95a6ac8c921f468a9283d7e309f97d2e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8feeb01d9c2e4ee893cf38f3c61df395, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/72a08964c05045f3bd0b453cd79b43f5, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b804a4407cf747499718f2fa68c1ee8c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b611c3ab2246444eaff0cd0116dfc079, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/386c5469af71438bad57cb72b16322d3, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/04767a49fbe146fcb1d7e6993ccb2a8b, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/7834543e9ac34e118b5e7111ea9fbb2c, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/0a945037ac734afe9d39d045706490de, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a230b8f2e931483e823d05d93afce977, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/ad49f684d597438f976bb3f1a354954a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/391e427995c54048ba63f6016b8289c8, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/6b3c95923e3940f08ceeeabdb10c17a8] to archive 2024-12-02T21:18:19,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:18:19,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/fa97b4586c8b4cf983ff890bc8a4af95 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/fa97b4586c8b4cf983ff890bc8a4af95 2024-12-02T21:18:19,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a989d6e06ca546e380330de726a1fa7a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a989d6e06ca546e380330de726a1fa7a 2024-12-02T21:18:19,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/5530a4537e464b1299f9ed41d122c35b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/5530a4537e464b1299f9ed41d122c35b 2024-12-02T21:18:19,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8e1d32a256dd4c2cb38ee69ab9f2f636 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8e1d32a256dd4c2cb38ee69ab9f2f636 2024-12-02T21:18:19,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/cfb02b910b8a4e498ec03788d3708a52 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/cfb02b910b8a4e498ec03788d3708a52 2024-12-02T21:18:19,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/f9e77bb21a33458194e1a5edd183cd10 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/f9e77bb21a33458194e1a5edd183cd10 2024-12-02T21:18:19,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/95a6ac8c921f468a9283d7e309f97d2e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/95a6ac8c921f468a9283d7e309f97d2e 2024-12-02T21:18:19,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8feeb01d9c2e4ee893cf38f3c61df395 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/8feeb01d9c2e4ee893cf38f3c61df395 2024-12-02T21:18:19,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/72a08964c05045f3bd0b453cd79b43f5 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/72a08964c05045f3bd0b453cd79b43f5 2024-12-02T21:18:19,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b804a4407cf747499718f2fa68c1ee8c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b804a4407cf747499718f2fa68c1ee8c 2024-12-02T21:18:19,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b611c3ab2246444eaff0cd0116dfc079 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/b611c3ab2246444eaff0cd0116dfc079 2024-12-02T21:18:19,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/386c5469af71438bad57cb72b16322d3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/386c5469af71438bad57cb72b16322d3 2024-12-02T21:18:19,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/04767a49fbe146fcb1d7e6993ccb2a8b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/04767a49fbe146fcb1d7e6993ccb2a8b 2024-12-02T21:18:19,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/7834543e9ac34e118b5e7111ea9fbb2c to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/7834543e9ac34e118b5e7111ea9fbb2c 2024-12-02T21:18:19,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/0a945037ac734afe9d39d045706490de to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/0a945037ac734afe9d39d045706490de 2024-12-02T21:18:19,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a230b8f2e931483e823d05d93afce977 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a230b8f2e931483e823d05d93afce977 2024-12-02T21:18:19,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/ad49f684d597438f976bb3f1a354954a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/ad49f684d597438f976bb3f1a354954a 2024-12-02T21:18:19,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/391e427995c54048ba63f6016b8289c8 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/391e427995c54048ba63f6016b8289c8 2024-12-02T21:18:19,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/6b3c95923e3940f08ceeeabdb10c17a8 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/6b3c95923e3940f08ceeeabdb10c17a8 2024-12-02T21:18:19,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ccd989354d6e4f84b06053910690d012, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/b52efdf85db64e5e8b0faf24cb0f105f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/48d261fb365d426393b5bed4f4271e7f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9161787da3634bfabdb0b97cf4726304, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5d0ebde9bf67432a99fecfd5aeb54df1, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/00a76cd790a949f4935185cf93577ae4, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/a2e99ff9af4f47b1a2d7940009e2c587, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/58e1b7cc3c92475a8863c8ba751cf3cf, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/41db991a8e674d65a56638652e6d5d3a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/85bd4ea8b4724533883d7341c4388efe, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac9d39388569473ab60ab3886b2f0e7f, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9729612020fe4e65b06290c687ecf581, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6100c758a5e1428799d8769881c8bd99, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6ace4064fc804f7c8ed5f80329eebfb7, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac2363db2fe74a3399b92d41740d068a, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5c601d41076a40e4b4352e44975c009d, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ff419eb67f8f4f8f86dd2d0f99564530, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eae87c35a28f4555a2b34f5b23e6781e, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eb750fd1e1f7479690f7c759afc0539b] to archive 2024-12-02T21:18:19,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T21:18:19,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ccd989354d6e4f84b06053910690d012 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ccd989354d6e4f84b06053910690d012 2024-12-02T21:18:19,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/b52efdf85db64e5e8b0faf24cb0f105f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/b52efdf85db64e5e8b0faf24cb0f105f 2024-12-02T21:18:19,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/48d261fb365d426393b5bed4f4271e7f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/48d261fb365d426393b5bed4f4271e7f 2024-12-02T21:18:19,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9161787da3634bfabdb0b97cf4726304 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9161787da3634bfabdb0b97cf4726304 2024-12-02T21:18:19,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5d0ebde9bf67432a99fecfd5aeb54df1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5d0ebde9bf67432a99fecfd5aeb54df1 2024-12-02T21:18:19,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/00a76cd790a949f4935185cf93577ae4 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/00a76cd790a949f4935185cf93577ae4 2024-12-02T21:18:19,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/a2e99ff9af4f47b1a2d7940009e2c587 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/a2e99ff9af4f47b1a2d7940009e2c587 2024-12-02T21:18:19,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/58e1b7cc3c92475a8863c8ba751cf3cf to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/58e1b7cc3c92475a8863c8ba751cf3cf 2024-12-02T21:18:19,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/41db991a8e674d65a56638652e6d5d3a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/41db991a8e674d65a56638652e6d5d3a 2024-12-02T21:18:19,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/85bd4ea8b4724533883d7341c4388efe to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/85bd4ea8b4724533883d7341c4388efe 2024-12-02T21:18:19,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac9d39388569473ab60ab3886b2f0e7f to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac9d39388569473ab60ab3886b2f0e7f 2024-12-02T21:18:19,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9729612020fe4e65b06290c687ecf581 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/9729612020fe4e65b06290c687ecf581 2024-12-02T21:18:19,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6100c758a5e1428799d8769881c8bd99 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6100c758a5e1428799d8769881c8bd99 2024-12-02T21:18:19,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6ace4064fc804f7c8ed5f80329eebfb7 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/6ace4064fc804f7c8ed5f80329eebfb7 2024-12-02T21:18:19,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac2363db2fe74a3399b92d41740d068a to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ac2363db2fe74a3399b92d41740d068a 2024-12-02T21:18:19,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5c601d41076a40e4b4352e44975c009d to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/5c601d41076a40e4b4352e44975c009d 2024-12-02T21:18:19,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ff419eb67f8f4f8f86dd2d0f99564530 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/ff419eb67f8f4f8f86dd2d0f99564530 2024-12-02T21:18:19,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eae87c35a28f4555a2b34f5b23e6781e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eae87c35a28f4555a2b34f5b23e6781e 2024-12-02T21:18:19,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eb750fd1e1f7479690f7c759afc0539b to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/eb750fd1e1f7479690f7c759afc0539b 2024-12-02T21:18:19,454 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/recovered.edits/318.seqid, newMaxSeqId=318, maxSeqId=4 2024-12-02T21:18:19,454 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. 2024-12-02T21:18:19,454 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 6cc723b24523dcb8953ed05d209a77e3: Waiting for close lock at 1733174297759Running coprocessor pre-close hooks at 1733174297759Disabling compacts and flushes for region at 1733174297759Disabling writes for close at 1733174297760 (+1 ms)Obtaining lock to block concurrent updates at 1733174297760Preparing flush snapshotting stores in 6cc723b24523dcb8953ed05d209a77e3 at 1733174297760Finished memstore snapshotting TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733174297760Flushing stores of TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3. at 1733174297760Flushing 6cc723b24523dcb8953ed05d209a77e3/A: creating writer at 1733174297760Flushing mob file 6cc723b24523dcb8953ed05d209a77e3/A: appending metadata at 1733174297764 (+4 ms)Flushing mob file 6cc723b24523dcb8953ed05d209a77e3/A: closing flushed file at 1733174297764Flushing 6cc723b24523dcb8953ed05d209a77e3/A: appending metadata at 1733174298171 (+407 ms)Flushing 6cc723b24523dcb8953ed05d209a77e3/A: closing flushed file at 1733174298171Flushing 6cc723b24523dcb8953ed05d209a77e3/B: creating writer at 1733174298578 (+407 ms)Flushing 6cc723b24523dcb8953ed05d209a77e3/B: appending metadata at 1733174298579 (+1 ms)Flushing 6cc723b24523dcb8953ed05d209a77e3/B: closing flushed file at 1733174298579Flushing 6cc723b24523dcb8953ed05d209a77e3/C: creating writer at 1733174298986 (+407 ms)Flushing 6cc723b24523dcb8953ed05d209a77e3/C: appending metadata at 1733174298987 (+1 ms)Flushing 6cc723b24523dcb8953ed05d209a77e3/C: closing flushed file at 1733174298987Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11c36601: reopening flushed file at 1733174299394 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cf3b544: reopening flushed file at 1733174299397 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52e3d193: reopening flushed file at 1733174299400 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6cc723b24523dcb8953ed05d209a77e3 in 1643ms, sequenceid=315, compaction requested=true at 1733174299403 (+3 ms)Writing region close event to WAL at 1733174299451 (+48 ms)Running coprocessor post-close hooks at 1733174299454 (+3 ms)Closed at 1733174299454 2024-12-02T21:18:19,456 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,456 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=6cc723b24523dcb8953ed05d209a77e3, regionState=CLOSED 2024-12-02T21:18:19,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 because future has completed 2024-12-02T21:18:19,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-02T21:18:19,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 6cc723b24523dcb8953ed05d209a77e3, server=87c3fdb6c570,35563,1733174113859 in 1.8490 sec 2024-12-02T21:18:19,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-12-02T21:18:19,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6cc723b24523dcb8953ed05d209a77e3, UNASSIGN in 1.8530 sec 2024-12-02T21:18:19,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-02T21:18:19,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8560 sec 2024-12-02T21:18:19,462 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174299462"}]},"ts":"1733174299462"} 2024-12-02T21:18:19,463 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T21:18:19,463 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T21:18:19,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8630 sec 2024-12-02T21:18:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T21:18:19,749 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:19,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T21:18:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:18:19,751 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:18:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T21:18:19,752 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:18:19,754 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,755 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C, FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/recovered.edits] 2024-12-02T21:18:19,808 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/1bc62ed450b9472b9cdccfd175900b75 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/1bc62ed450b9472b9cdccfd175900b75 2024-12-02T21:18:19,810 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/2afc97cb75de4cabaeb2b1678fcb510e to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/2afc97cb75de4cabaeb2b1678fcb510e 2024-12-02T21:18:19,811 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/66bd1e31d43046b2abc29c3dde7da6a1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/A/66bd1e31d43046b2abc29c3dde7da6a1 2024-12-02T21:18:19,813 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/392cd8835d634f94b694d52a8b909f40 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/392cd8835d634f94b694d52a8b909f40 2024-12-02T21:18:19,813 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/78be44634b5c44c7b8dbd9616dfa3b66 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/78be44634b5c44c7b8dbd9616dfa3b66 2024-12-02T21:18:19,814 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a7d95725fe224bbda3f497550b2465b2 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/B/a7d95725fe224bbda3f497550b2465b2 2024-12-02T21:18:19,816 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/25bde5e7491a4ffc82877d35d82e0dab to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/25bde5e7491a4ffc82877d35d82e0dab 2024-12-02T21:18:19,816 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/3c05ccd45a4244e287ca7c485852aee1 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/3c05ccd45a4244e287ca7c485852aee1 2024-12-02T21:18:19,817 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/7d783eebdc254512b43429069d39fecd to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/C/7d783eebdc254512b43429069d39fecd 2024-12-02T21:18:19,818 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/recovered.edits/318.seqid to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3/recovered.edits/318.seqid 2024-12-02T21:18:19,819 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/default/TestAcidGuarantees/6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,819 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T21:18:19,819 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T21:18:19,820 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T21:18:19,821 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202040b4ae7bd474b27abe956a9ec29f892_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202040b4ae7bd474b27abe956a9ec29f892_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,822 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202061ffcd2329a4e64929ef19c259a5a8a_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202061ffcd2329a4e64929ef19c259a5a8a_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,823 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120209d502f35e454ec0ac082de4dd98c635_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120209d502f35e454ec0ac082de4dd98c635_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,823 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202120f7533e37e4c6896e3d7c5ffd9857c_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202120f7533e37e4c6896e3d7c5ffd9857c_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,824 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021c11d088a657402f8222c57355c271cf_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021c11d088a657402f8222c57355c271cf_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,825 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024fd0b8b2b0d64b128b1fbe68fe18596e_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024fd0b8b2b0d64b128b1fbe68fe18596e_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,826 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120257070e1e3f1b421bb3da984eee98794e_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120257070e1e3f1b421bb3da984eee98794e_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,826 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025b1f9824fb7049a1a515be9e31f517dc_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025b1f9824fb7049a1a515be9e31f517dc_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,827 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025ec0d57eaab64a818ad47fd702b462fc_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025ec0d57eaab64a818ad47fd702b462fc_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,828 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120274494a9ea5ae465b8e7fe9eb0df116bd_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120274494a9ea5ae465b8e7fe9eb0df116bd_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,828 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202778edcdc0b844f2b804adf973bcb701d_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202778edcdc0b844f2b804adf973bcb701d_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,829 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120284b91adb9621494abea5f55190572b07_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120284b91adb9621494abea5f55190572b07_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,830 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028aa3ded80f9c4bd3ab56114d1050ec10_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028aa3ded80f9c4bd3ab56114d1050ec10_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,831 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202baa288abccb6490db34a59245b47b9b0_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202baa288abccb6490db34a59245b47b9b0_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,831 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bf5f7a1f5a724067ba9882a0f754678b_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bf5f7a1f5a724067ba9882a0f754678b_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,832 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202df8e362089354819b6c1a4e78dd62320_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202df8e362089354819b6c1a4e78dd62320_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,833 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202feb1a02c6bbe439ebffc21be33d7699a_6cc723b24523dcb8953ed05d209a77e3 to hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202feb1a02c6bbe439ebffc21be33d7699a_6cc723b24523dcb8953ed05d209a77e3 2024-12-02T21:18:19,833 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T21:18:19,835 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:18:19,837 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T21:18:19,838 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T21:18:19,839 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:18:19,839 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T21:18:19,840 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733174299839"}]},"ts":"9223372036854775807"} 2024-12-02T21:18:19,841 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T21:18:19,841 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6cc723b24523dcb8953ed05d209a77e3, NAME => 'TestAcidGuarantees,,1733174271464.6cc723b24523dcb8953ed05d209a77e3.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T21:18:19,841 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T21:18:19,841 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733174299841"}]},"ts":"9223372036854775807"} 2024-12-02T21:18:19,842 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T21:18:19,843 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T21:18:19,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-12-02T21:18:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T21:18:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T21:18:19,859 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T21:18:19,868 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 246) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=475 (was 441) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1846 (was 1957) 2024-12-02T21:18:19,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T21:18:19,868 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:18:19,868 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:18:19,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:19,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:19,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:18:19,868 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T21:18:19,868 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1866486918, stopped=false 2024-12-02T21:18:19,869 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=87c3fdb6c570,39577,1733174113031 2024-12-02T21:18:19,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T21:18:19,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T21:18:19,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:18:19,998 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:18:19,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:18:20,001 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:18:20,002 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:18:20,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:20,002 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:18:20,002 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:18:20,002 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '87c3fdb6c570,35563,1733174113859' ***** 2024-12-02T21:18:20,002 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T21:18:20,002 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T21:18:20,003 INFO [RS:0;87c3fdb6c570:35563 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T21:18:20,003 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T21:18:20,003 INFO [RS:0;87c3fdb6c570:35563 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T21:18:20,003 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(959): stopping server 87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:20,003 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T21:18:20,003 INFO [RS:0;87c3fdb6c570:35563 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;87c3fdb6c570:35563. 2024-12-02T21:18:20,003 DEBUG [RS:0;87c3fdb6c570:35563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:18:20,003 DEBUG [RS:0;87c3fdb6c570:35563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:20,003 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T21:18:20,003 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T21:18:20,003 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T21:18:20,003 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T21:18:20,004 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-02T21:18:20,004 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-02T21:18:20,004 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T21:18:20,004 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T21:18:20,004 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T21:18:20,004 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T21:18:20,004 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T21:18:20,004 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T21:18:20,004 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-02T21:18:20,025 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/info/bdb482d592ee4f5dac2b126ead60cd38 is 95, key is TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c./info:/1733174145353/DeleteFamily/seqid=0 2024-12-02T21:18:20,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742412_1588 (size=6514) 2024-12-02T21:18:20,053 INFO [regionserver/87c3fdb6c570:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T21:18:20,204 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T21:18:20,404 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T21:18:20,428 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/info/bdb482d592ee4f5dac2b126ead60cd38 2024-12-02T21:18:20,448 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/ns/149e85e9787a428782969b4b257a0d14 is 93, key is TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c./ns:/1733174145314/DeleteFamily/seqid=0 2024-12-02T21:18:20,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742413_1589 (size=5906) 2024-12-02T21:18:20,605 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T21:18:20,805 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T21:18:20,851 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/ns/149e85e9787a428782969b4b257a0d14 2024-12-02T21:18:20,866 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/rep_barrier/b9bfaaafbd5240219bcf4f4ce1c4cd56 is 102, key is TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c./rep_barrier:/1733174145314/DeleteFamily/seqid=0 2024-12-02T21:18:20,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742414_1590 (size=6025) 2024-12-02T21:18:20,916 INFO [regionserver/87c3fdb6c570:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T21:18:20,916 INFO [regionserver/87c3fdb6c570:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T21:18:21,005 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T21:18:21,205 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T21:18:21,269 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/rep_barrier/b9bfaaafbd5240219bcf4f4ce1c4cd56 2024-12-02T21:18:21,283 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/table/aeeb8e7246e84c499f56d26f0ff11fa5 is 96, key is TestAcidGuarantees,,1733174117560.8a7bf3289fb638014baa7fb12fa0d82c./table:/1733174145314/DeleteFamily/seqid=0 2024-12-02T21:18:21,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742415_1591 (size=5936) 2024-12-02T21:18:21,405 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T21:18:21,605 DEBUG [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T21:18:21,687 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/table/aeeb8e7246e84c499f56d26f0ff11fa5 2024-12-02T21:18:21,691 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/info/bdb482d592ee4f5dac2b126ead60cd38 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/info/bdb482d592ee4f5dac2b126ead60cd38 2024-12-02T21:18:21,694 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/info/bdb482d592ee4f5dac2b126ead60cd38, entries=12, sequenceid=90, filesize=6.4 K 2024-12-02T21:18:21,695 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/ns/149e85e9787a428782969b4b257a0d14 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/ns/149e85e9787a428782969b4b257a0d14 2024-12-02T21:18:21,698 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/ns/149e85e9787a428782969b4b257a0d14, entries=8, sequenceid=90, filesize=5.8 K 2024-12-02T21:18:21,698 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/rep_barrier/b9bfaaafbd5240219bcf4f4ce1c4cd56 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/rep_barrier/b9bfaaafbd5240219bcf4f4ce1c4cd56 2024-12-02T21:18:21,701 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/rep_barrier/b9bfaaafbd5240219bcf4f4ce1c4cd56, entries=6, sequenceid=90, filesize=5.9 K 2024-12-02T21:18:21,701 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/.tmp/table/aeeb8e7246e84c499f56d26f0ff11fa5 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/table/aeeb8e7246e84c499f56d26f0ff11fa5 2024-12-02T21:18:21,704 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/table/aeeb8e7246e84c499f56d26f0ff11fa5, entries=7, sequenceid=90, filesize=5.8 K 2024-12-02T21:18:21,704 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1700ms, sequenceid=90, compaction requested=false 2024-12-02T21:18:21,709 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-02T21:18:21,710 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T21:18:21,710 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T21:18:21,710 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733174300004Running coprocessor pre-close hooks at 1733174300004Disabling compacts and flushes for region at 1733174300004Disabling writes for close at 1733174300004Obtaining lock to block concurrent updates at 1733174300004Preparing flush snapshotting stores in 1588230740 at 1733174300004Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733174300005 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733174300005Flushing 1588230740/info: creating writer at 1733174300005Flushing 1588230740/info: appending metadata at 1733174300024 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733174300024Flushing 1588230740/ns: creating writer at 1733174300432 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733174300447 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733174300448 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1733174300854 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733174300865 (+11 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733174300865Flushing 1588230740/table: creating writer at 1733174301272 (+407 ms)Flushing 1588230740/table: appending metadata at 1733174301283 (+11 ms)Flushing 1588230740/table: closing flushed file at 1733174301283Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6101a00d: reopening flushed file at 1733174301690 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24ed2646: reopening flushed file at 1733174301694 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@645c0d2f: reopening flushed file at 1733174301698 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@506f2543: reopening flushed file at 1733174301701 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1700ms, sequenceid=90, compaction requested=false at 1733174301704 (+3 ms)Writing region close event to WAL at 1733174301707 (+3 ms)Running coprocessor post-close hooks at 1733174301710 (+3 ms)Closed at 1733174301710 2024-12-02T21:18:21,710 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T21:18:21,806 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(976): stopping server 87c3fdb6c570,35563,1733174113859; all regions closed. 2024-12-02T21:18:21,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741834_1010 (size=25344) 2024-12-02T21:18:21,811 DEBUG [RS:0;87c3fdb6c570:35563 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/oldWALs 2024-12-02T21:18:21,812 INFO [RS:0;87c3fdb6c570:35563 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 87c3fdb6c570%2C35563%2C1733174113859.meta:.meta(num 1733174117136) 2024-12-02T21:18:21,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741832_1008 (size=22444779) 2024-12-02T21:18:21,814 DEBUG [RS:0;87c3fdb6c570:35563 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/oldWALs 2024-12-02T21:18:21,814 INFO [RS:0;87c3fdb6c570:35563 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 87c3fdb6c570%2C35563%2C1733174113859:(num 1733174116070) 2024-12-02T21:18:21,815 DEBUG [RS:0;87c3fdb6c570:35563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:18:21,815 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T21:18:21,815 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T21:18:21,815 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.ChoreService(370): Chore service for: regionserver/87c3fdb6c570:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T21:18:21,815 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T21:18:21,815 INFO [regionserver/87c3fdb6c570:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T21:18:21,815 INFO [RS:0;87c3fdb6c570:35563 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35563 2024-12-02T21:18:21,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/87c3fdb6c570,35563,1733174113859 2024-12-02T21:18:21,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T21:18:21,871 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T21:18:21,872 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [87c3fdb6c570,35563,1733174113859] 2024-12-02T21:18:21,882 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/87c3fdb6c570,35563,1733174113859 already deleted, retry=false 2024-12-02T21:18:21,882 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 87c3fdb6c570,35563,1733174113859 expired; onlineServers=0 2024-12-02T21:18:21,882 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '87c3fdb6c570,39577,1733174113031' ***** 2024-12-02T21:18:21,882 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T21:18:21,882 INFO [M:0;87c3fdb6c570:39577 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T21:18:21,882 INFO [M:0;87c3fdb6c570:39577 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T21:18:21,882 DEBUG [M:0;87c3fdb6c570:39577 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T21:18:21,882 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T21:18:21,882 DEBUG [M:0;87c3fdb6c570:39577 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T21:18:21,882 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.small.0-1733174115807 {}] cleaner.HFileCleaner(306): Exit Thread[master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.small.0-1733174115807,5,FailOnTimeoutGroup] 2024-12-02T21:18:21,882 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.large.0-1733174115806 {}] cleaner.HFileCleaner(306): Exit Thread[master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.large.0-1733174115806,5,FailOnTimeoutGroup] 2024-12-02T21:18:21,883 INFO [M:0;87c3fdb6c570:39577 {}] hbase.ChoreService(370): Chore service for: master/87c3fdb6c570:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T21:18:21,883 INFO [M:0;87c3fdb6c570:39577 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T21:18:21,883 DEBUG [M:0;87c3fdb6c570:39577 {}] master.HMaster(1795): Stopping service threads 2024-12-02T21:18:21,883 INFO [M:0;87c3fdb6c570:39577 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T21:18:21,883 INFO [M:0;87c3fdb6c570:39577 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:18:21,883 ERROR [M:0;87c3fdb6c570:39577 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-02T21:18:21,884 INFO [M:0;87c3fdb6c570:39577 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:18:21,884 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T21:18:21,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T21:18:21,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:18:21,892 DEBUG [M:0;87c3fdb6c570:39577 {}] zookeeper.ZKUtil(347): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T21:18:21,892 WARN [M:0;87c3fdb6c570:39577 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T21:18:21,893 INFO [M:0;87c3fdb6c570:39577 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/.lastflushedseqids 2024-12-02T21:18:21,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742416_1592 (size=72) 2024-12-02T21:18:21,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T21:18:21,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35563-0x10197e62b900001, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T21:18:21,983 INFO [RS:0;87c3fdb6c570:35563 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T21:18:21,983 INFO [RS:0;87c3fdb6c570:35563 {}] regionserver.HRegionServer(1031): Exiting; stopping=87c3fdb6c570,35563,1733174113859; zookeeper connection closed. 2024-12-02T21:18:21,983 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75ef038c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75ef038c 2024-12-02T21:18:21,983 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T21:18:22,302 INFO [M:0;87c3fdb6c570:39577 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:18:22,303 INFO [M:0;87c3fdb6c570:39577 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T21:18:22,303 DEBUG [M:0;87c3fdb6c570:39577 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T21:18:22,303 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:18:22,303 DEBUG [M:0;87c3fdb6c570:39577 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:18:22,303 DEBUG [M:0;87c3fdb6c570:39577 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T21:18:22,303 DEBUG [M:0;87c3fdb6c570:39577 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:18:22,303 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=773.95 KB heapSize=948.30 KB 2024-12-02T21:18:22,315 DEBUG [M:0;87c3fdb6c570:39577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/22ec6f901da143df84ce69155023bda6 is 82, key is hbase:meta,,1/info:regioninfo/1733174117218/Put/seqid=0 2024-12-02T21:18:22,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742417_1593 (size=5672) 2024-12-02T21:18:22,718 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2147 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/22ec6f901da143df84ce69155023bda6 2024-12-02T21:18:22,741 DEBUG [M:0;87c3fdb6c570:39577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2d2a80929344cca86f3d340a3304ac9 is 2479, key is \x00\x00\x00\x00\x00\x00\x00Z/proc:d/1733174206491/Put/seqid=0 2024-12-02T21:18:22,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742418_1594 (size=46043) 2024-12-02T21:18:23,147 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=773.39 KB at sequenceid=2147 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2d2a80929344cca86f3d340a3304ac9 2024-12-02T21:18:23,150 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2d2a80929344cca86f3d340a3304ac9 2024-12-02T21:18:23,163 DEBUG [M:0;87c3fdb6c570:39577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a13df802ada24fa1bd9ac314fe61b3b7 is 69, key is 87c3fdb6c570,35563,1733174113859/rs:state/1733174115843/Put/seqid=0 2024-12-02T21:18:23,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073742419_1595 (size=5156) 2024-12-02T21:18:23,356 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T21:18:23,357 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T21:18:23,358 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T21:18:23,566 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2147 (bloomFilter=true), to=hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a13df802ada24fa1bd9ac314fe61b3b7 2024-12-02T21:18:23,569 DEBUG [M:0;87c3fdb6c570:39577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/22ec6f901da143df84ce69155023bda6 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/22ec6f901da143df84ce69155023bda6 2024-12-02T21:18:23,572 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/22ec6f901da143df84ce69155023bda6, entries=8, sequenceid=2147, filesize=5.5 K 2024-12-02T21:18:23,573 DEBUG [M:0;87c3fdb6c570:39577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2d2a80929344cca86f3d340a3304ac9 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2d2a80929344cca86f3d340a3304ac9 2024-12-02T21:18:23,576 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2d2a80929344cca86f3d340a3304ac9 2024-12-02T21:18:23,576 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2d2a80929344cca86f3d340a3304ac9, entries=174, sequenceid=2147, filesize=45.0 K 2024-12-02T21:18:23,577 DEBUG [M:0;87c3fdb6c570:39577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a13df802ada24fa1bd9ac314fe61b3b7 as hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a13df802ada24fa1bd9ac314fe61b3b7 2024-12-02T21:18:23,579 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46549/user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a13df802ada24fa1bd9ac314fe61b3b7, entries=1, sequenceid=2147, filesize=5.0 K 2024-12-02T21:18:23,579 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.HRegion(3140): Finished flush of dataSize ~773.95 KB/792525, heapSize ~948 KB/970752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2147, compaction requested=false 2024-12-02T21:18:23,581 INFO [M:0;87c3fdb6c570:39577 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:18:23,581 DEBUG [M:0;87c3fdb6c570:39577 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733174302303Disabling compacts and flushes for region at 1733174302303Disabling writes for close at 1733174302303Obtaining lock to block concurrent updates at 1733174302303Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733174302303Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=792525, getHeapSize=970992, getOffHeapSize=0, getCellsCount=2346 at 1733174302303Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733174302304 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733174302304Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733174302315 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733174302315Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733174302721 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733174302741 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733174302741Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733174303150 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733174303163 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733174303163Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@409dc288: reopening flushed file at 1733174303569 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c28d80e: reopening flushed file at 1733174303573 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b30c4df: reopening flushed file at 1733174303576 (+3 ms)Finished flush of dataSize ~773.95 KB/792525, heapSize ~948 KB/970752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2147, compaction requested=false at 1733174303579 (+3 ms)Writing region close event to WAL at 1733174303581 (+2 ms)Closed at 1733174303581 2024-12-02T21:18:23,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39153 is added to blk_1073741830_1006 (size=933295) 2024-12-02T21:18:23,583 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/6409d617-dd3e-b233-ae5f-5aee3a5183e0/MasterData/WALs/87c3fdb6c570,39577,1733174113031/87c3fdb6c570%2C39577%2C1733174113031.1733174115119 not finished, retry = 0 2024-12-02T21:18:23,684 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T21:18:23,684 INFO [M:0;87c3fdb6c570:39577 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T21:18:23,684 INFO [M:0;87c3fdb6c570:39577 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39577 2024-12-02T21:18:23,684 INFO [M:0;87c3fdb6c570:39577 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T21:18:23,834 INFO [M:0;87c3fdb6c570:39577 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T21:18:23,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T21:18:23,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39577-0x10197e62b900000, quorum=127.0.0.1:53072, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T21:18:23,838 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T21:18:23,840 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T21:18:23,840 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T21:18:23,841 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T21:18:23,841 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/hadoop.log.dir/,STOPPED} 2024-12-02T21:18:23,843 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T21:18:23,843 WARN [BP-855536078-172.17.0.3-1733174109311 heartbeating to localhost/127.0.0.1:46549 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T21:18:23,843 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T21:18:23,843 WARN [BP-855536078-172.17.0.3-1733174109311 heartbeating to localhost/127.0.0.1:46549 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-855536078-172.17.0.3-1733174109311 (Datanode Uuid 75b38a8c-0eb4-4a7c-a097-bbbfabaebe5d) service to localhost/127.0.0.1:46549 2024-12-02T21:18:23,845 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52/data/data1/current/BP-855536078-172.17.0.3-1733174109311 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T21:18:23,845 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/cluster_6998bf75-9aa3-2c97-e61f-429ad3ee4f52/data/data2/current/BP-855536078-172.17.0.3-1733174109311 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T21:18:23,846 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T21:18:23,852 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T21:18:23,853 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T21:18:23,853 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T21:18:23,853 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T21:18:23,853 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21046b1f-2a11-3d1e-b22c-91033f880630/hadoop.log.dir/,STOPPED} 2024-12-02T21:18:23,865 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T21:18:23,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down